| Message ID | 20231130161624.1833206-1-andrey.popov@yadro.com |
|---|---|
| State | New |
| Headers | show |
| Series | [v3] image-live: Add support for building EFI-bootable ISO images for non-x86-based archs | expand |
Hello, This fails on the AB: https://autobuilder.yoctoproject.org/typhoon/#/builders/58/builds/8051/steps/12/logs/errors ERROR: Nothing PROVIDES 'libisoburn-native'. On 30/11/2023 19:16:24+0300, Andrey Popov wrote: > Since syslinux is only compatible with platforms that use x86-based > CPUs, this change allows creation of bootable ISO images for other > EFI-compatible platforms by replacing invocation of the isohybrid > tool for those platforms with a python script that creates MBR > partition table with a single entry that points to a bootable > EFI image placed inside the ISO image. > > Signed-off-by: Andrey Popov <andrey.popov@yadro.com> > --- > meta/classes-recipe/image-live.bbclass | 124 ++++++++++++++++++------- > 1 file changed, 92 insertions(+), 32 deletions(-) > > diff --git a/meta/classes-recipe/image-live.bbclass b/meta/classes-recipe/image-live.bbclass > index 95dd44a..c01bc71 100644 > --- a/meta/classes-recipe/image-live.bbclass > +++ b/meta/classes-recipe/image-live.bbclass > @@ -2,15 +2,15 @@ > # > # SPDX-License-Identifier: MIT > > -# Creates a bootable image using syslinux, your kernel and an optional > +# Creates a bootable image using syslinux (for x86), your kernel and an optional > # initrd > > # > # End result is two things: > # > -# 1. A .hddimg file which is an msdos filesystem containing syslinux, a kernel, > -# an initrd and a rootfs image. These can be written to harddisks directly and > -# also booted on USB flash disks (write them there with dd). > +# 1. A .hddimg file which is an msdos filesystem containing syslinux (for x86), > +# a kernel, an initrd and a rootfs image. These can be written to harddisks > +# directly and also booted on USB flash disks (write them there with dd). > # > # 2. A CD .iso image > > @@ -18,6 +18,10 @@ > # in syslinux. Actions based on the label are then performed (e.g. installing to > # an hdd) > > +# Since some versions of isohybrid are still looking for the isolinux.bin inside > +# the target image, if machine does not have pcbios feature defined we have to > +# use our own method to create EFI-bootable MBR. > + > # External variables (also used by syslinux.bbclass) > # ${INITRD} - indicates a list of filesystem images to concatenate and use as an initrd (optional) > # ${HDDIMG_ID} - FAT image volume-id > @@ -29,8 +33,12 @@ do_bootimg[depends] += "dosfstools-native:do_populate_sysroot \ > mtools-native:do_populate_sysroot \ > cdrtools-native:do_populate_sysroot \ > virtual/kernel:do_deploy \ > - ${MLPREFIX}syslinux:do_populate_sysroot \ > - syslinux-native:do_populate_sysroot \ > + ${@bb.utils.contains('MACHINE_FEATURES', 'pcbios', \ > + d.getVar('MLPREFIX') + 'syslinux:do_populate_sysroot \ > + syslinux-native:do_populate_sysroot', '', d)} \ > + ${@bb.utils.contains('MACHINE_FEATURES', 'efi', \ > + bb.utils.contains('MACHINE_FEATURES', 'pcbios', \ > + '', 'libisoburn-native:do_populate_sysroot', d), '', d)} \ > ${@'%s:do_image_%s' % (d.getVar('PN'), d.getVar('LIVE_ROOTFS_TYPE').replace('-', '_')) if d.getVar('ROOTFS') else ''} \ > " > > @@ -66,19 +74,65 @@ COMPACT_ISODIR = "${S}/iso.z" > ISOLINUXDIR ?= "/isolinux" > ISO_BOOTIMG = "isolinux/isolinux.bin" > ISO_BOOTCAT = "isolinux/boot.cat" > -MKISOFS_OPTIONS = "-no-emul-boot -boot-load-size 4 -boot-info-table" > +MKISOFS_BOOT_OPTIONS = "-no-emul-boot -boot-load-size 4 -boot-info-table" > > BOOTIMG_VOLUME_ID ?= "boot" > BOOTIMG_EXTRA_SPACE ?= "512" > > +def compute_chs(sector_z): > + C = int(sector_z / (63 * 255)) > + H = int((sector_z % (63 * 255)) / 63) > + # convert zero-based sector to CHS format > + S = int(sector_z % 63) + 1 > + # munge accord to partition table format > + S = (S & 0x3f) | (((C >> 8) & 0x3) << 6) > + C = (C & 0xFF) > + return (C, H, S) > + > +def mk_efi_part_table(iso, start, length): > + from struct import pack > + > + # Compute starting and ending CHS addresses for the partition entry. > + (s_C, s_H, s_S) = compute_chs(start) > + (e_C, e_H, e_S) = compute_chs(start + length - 1) > + > + # Write the 66 byte partition table to bytes 0x1BE through 0x1FF in > + # sector 0 of the .ISO. > + # > + # See the partition table format here: > + # http://en.wikipedia.org/wiki/Master_boot_record#Sector_layout > + f = open(iso, 'r+b') > + f.seek(0x1BE) > + f.write(pack("<8BLL48xH", 0x80, s_H, s_S, s_C, > + 0xEF, e_H, e_S, e_C, start, length, 0xAA55)) > + f.close() > + > +def install_efi_part_table(iso_img): > + import subprocess > + > + find_efi_img_cmd = "xorriso -indev %s -find /efi.img \ > + -name efi.img -exec report_lba --" % iso_img > + ret = subprocess.run(find_efi_img_cmd.split(), capture_output=True) > + efi_img_start = -1 > + efi_img_length = -1 > + for line in ret.stdout.decode(encoding='utf-8').split("\n"): > + if "File data lba:" in line and "/efi.img" in line: > + file_stat = line[14:].split(',') > + efi_img_start = int(file_stat[1].strip()) * 4 > + efi_img_length = int(int(file_stat[3].strip()) / 512) > + break > + if (efi_img_start < 0) or (efi_img_length < 0): > + bb.fatal("Failed to determine /efi.img attributes") > + mk_efi_part_table(iso_img, efi_img_start, efi_img_length) > + > populate_live() { > - populate_kernel $1 > + populate_kernel $1 > if [ -s "${ROOTFS}" ]; then > install -m 0644 ${ROOTFS} $1/rootfs.img > fi > } > > -build_iso() { > +build_iso_base() { > # Only create an ISO if we have an INITRD and the live or iso image type was selected > if [ -z "${INITRD}" ] || [ "${@bb.utils.contains_any('IMAGE_FSTYPES', 'live iso', '1', '0', d)}" != "1" ]; then > bbnote "ISO image will not be created." > @@ -103,14 +157,6 @@ build_iso() { > build_fat_img ${EFIIMGDIR} ${ISODIR}/efi.img > fi > > - # EFI only > - if [ "${PCBIOS}" != "1" ] && [ "${EFI}" = "1" ] ; then > - # Work around bug in isohybrid where it requires isolinux.bin > - # In the boot catalog, even though it is not used > - mkdir -p ${ISODIR}/${ISOLINUXDIR} > - install -m 0644 ${STAGING_DATADIR}/syslinux/isolinux.bin ${ISODIR}${ISOLINUXDIR} > - fi > - > # We used to have support for zisofs; this is a relic of that > mkisofs_compress_opts="-r" > > @@ -128,26 +174,40 @@ build_iso() { > fi > fi > > - if [ "${PCBIOS}" = "1" ] && [ "${EFI}" != "1" ] ; then > - # PCBIOS only media > - mkisofs -V ${BOOTIMG_VOLUME_ID} \ > - -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ > - -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ > - $mkisofs_compress_opts \ > - ${MKISOFS_OPTIONS} $mkisofs_iso_level ${ISODIR} > + if [ "${PCBIOS}" = "1" ]; then > + if [ "${EFI}" = "1" ]; then > + # EFI+PCBIOS > + mkisofs -A ${BOOTIMG_VOLUME_ID} -V ${BOOTIMG_VOLUME_ID} \ > + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ > + -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ > + $mkisofs_compress_opts ${MKISOFS_BOOT_OPTIONS} $mkisofs_iso_level \ > + -eltorito-alt-boot -eltorito-platform efi \ > + -b efi.img -no-emul-boot \ > + ${ISODIR} > + isohybrid_args="-u" > + else > + # PCBIOS only > + mkisofs -V ${BOOTIMG_VOLUME_ID} \ > + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ > + -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ > + $mkisofs_compress_opts \ > + ${MKISOFS_BOOT_OPTIONS} $mkisofs_iso_level ${ISODIR} > + fi > + > + isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso > else > - # EFI only OR EFI+PCBIOS > mkisofs -A ${BOOTIMG_VOLUME_ID} -V ${BOOTIMG_VOLUME_ID} \ > - -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ > - -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ > - $mkisofs_compress_opts ${MKISOFS_OPTIONS} $mkisofs_iso_level \ > - -eltorito-alt-boot -eltorito-platform efi \ > - -b efi.img -no-emul-boot \ > + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ > + $mkisofs_compress_opts $mkisofs_iso_level \ > ${ISODIR} > - isohybrid_args="-u" > fi > +} > > - isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso > +python build_iso() { > + bb.build.exec_func("build_iso_base", d) > + if d.getVar("PCBIOS") != "1" and d.getVar("EFI") == "1": > + install_efi_part_table(d.getVar("IMGDEPLOYDIR") + "/" + \ > + d.getVar("IMAGE_NAME") + ".iso") > } > > build_fat_img() { > -- > 2.34.1 > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#191512): https://lists.openembedded.org/g/openembedded-core/message/191512 > Mute This Topic: https://lists.openembedded.org/mt/102896781/3617179 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alexandre.belloni@bootlin.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Hello, Alexandre! Thank you for the feedback. I think I can eliminate the xorriso/libisoburn dependency by replacing it with a python script that I've created which uses modified version of isoparser python package. However, I am not aware of the correct way to include this tool into openembedded-core. If someone can suggest such a way (if it is a good idea after all), that would be greatly appreciated. Best regards, Andrey.
On Thu, 30 Nov 2023 at 17:16, Andrey Popov <andrey.popov@yadro.com> wrote: > - # EFI only > - if [ "${PCBIOS}" != "1" ] && [ "${EFI}" = "1" ] ; then > - # Work around bug in isohybrid where it requires isolinux.bin > - # In the boot catalog, even though it is not used > - mkdir -p ${ISODIR}/${ISOLINUXDIR} > - install -m 0644 ${STAGING_DATADIR}/syslinux/isolinux.bin ${ISODIR}${ISOLINUXDIR} > - fi > - Why is this snippet removed? I don't see anything that replaces it. If the removal is valid, do we still need syslinux? Alex
Hello, Alex! With this patch applied, targets that request image-live and do not define pcbios feature should not depend on syslinux anymore. However, this patch requires an external tool (or, at least, isoparser python package to be imported inside the class) to determine the efi.img offset and size inside ISO image. Regarding the snippet in question, it can be safely omitted since it was required for isohybrid to work for targets that define efi feature but do not define pcbios one, and because this patch replaces invocation of this tool with custom method for creating EFI-bootable MBR in aforementioned case. Best regards, Andrey.
How big is the script? Can it be 'inlined' into the class as a set of functions? Alex On Mon, 4 Dec 2023 at 14:33, Andrey Popov <andrey.popov@yadro.com> wrote: > > Hello, Alexandre! > > Thank you for the feedback. > I think I can eliminate the xorriso/libisoburn dependency by replacing it with a python script that I've created which uses modified version of isoparser python package. > However, I am not aware of the correct way to include this tool into openembedded-core. If someone can suggest such a way (if it is a good idea after all), that would be greatly appreciated. > > Best regards, Andrey. > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#191737): https://lists.openembedded.org/g/openembedded-core/message/191737 > Mute This Topic: https://lists.openembedded.org/mt/102896781/1686489 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
Hello, Alex! I have doubts that the entire isoparser class set could be inlined, that would occupy around 1200 lines in total. https://github.com/barneygale/isoparser/tree/master/isoparser Three functions that are used to find the file using this class set are 32 lines long in total. Best regards, Andrey.
Then it's probably best to write a recipe, so that the code can be hosted externally and maintained independently of oe-core? Alex On Mon, 4 Dec 2023 at 15:46, Andrey Popov <andrey.popov@yadro.com> wrote: > > Hello, Alex! > > I have doubts that the entire isoparser class set could be inlined, that would occupy around 1200 lines in total. > https://github.com/barneygale/isoparser/tree/master/isoparser > Three functions that are used to find the file using this class set are 32 lines long in total. > > Best regards, Andrey. > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#191742): https://lists.openembedded.org/g/openembedded-core/message/191742 > Mute This Topic: https://lists.openembedded.org/mt/102896781/1686489 > Group Owner: openembedded-core+owner@lists.openembedded.org > Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [alex.kanavin@gmail.com] > -=-=-=-=-=-=-=-=-=-=-=- >
diff --git a/meta/classes-recipe/image-live.bbclass b/meta/classes-recipe/image-live.bbclass index 95dd44a..c01bc71 100644 --- a/meta/classes-recipe/image-live.bbclass +++ b/meta/classes-recipe/image-live.bbclass @@ -2,15 +2,15 @@ # # SPDX-License-Identifier: MIT -# Creates a bootable image using syslinux, your kernel and an optional +# Creates a bootable image using syslinux (for x86), your kernel and an optional # initrd # # End result is two things: # -# 1. A .hddimg file which is an msdos filesystem containing syslinux, a kernel, -# an initrd and a rootfs image. These can be written to harddisks directly and -# also booted on USB flash disks (write them there with dd). +# 1. A .hddimg file which is an msdos filesystem containing syslinux (for x86), +# a kernel, an initrd and a rootfs image. These can be written to harddisks +# directly and also booted on USB flash disks (write them there with dd). # # 2. A CD .iso image @@ -18,6 +18,10 @@ # in syslinux. Actions based on the label are then performed (e.g. installing to # an hdd) +# Since some versions of isohybrid are still looking for the isolinux.bin inside +# the target image, if machine does not have pcbios feature defined we have to +# use our own method to create EFI-bootable MBR. + # External variables (also used by syslinux.bbclass) # ${INITRD} - indicates a list of filesystem images to concatenate and use as an initrd (optional) # ${HDDIMG_ID} - FAT image volume-id @@ -29,8 +33,12 @@ do_bootimg[depends] += "dosfstools-native:do_populate_sysroot \ mtools-native:do_populate_sysroot \ cdrtools-native:do_populate_sysroot \ virtual/kernel:do_deploy \ - ${MLPREFIX}syslinux:do_populate_sysroot \ - syslinux-native:do_populate_sysroot \ + ${@bb.utils.contains('MACHINE_FEATURES', 'pcbios', \ + d.getVar('MLPREFIX') + 'syslinux:do_populate_sysroot \ + syslinux-native:do_populate_sysroot', '', d)} \ + ${@bb.utils.contains('MACHINE_FEATURES', 'efi', \ + bb.utils.contains('MACHINE_FEATURES', 'pcbios', \ + '', 'libisoburn-native:do_populate_sysroot', d), '', d)} \ ${@'%s:do_image_%s' % (d.getVar('PN'), d.getVar('LIVE_ROOTFS_TYPE').replace('-', '_')) if d.getVar('ROOTFS') else ''} \ " @@ -66,19 +74,65 @@ COMPACT_ISODIR = "${S}/iso.z" ISOLINUXDIR ?= "/isolinux" ISO_BOOTIMG = "isolinux/isolinux.bin" ISO_BOOTCAT = "isolinux/boot.cat" -MKISOFS_OPTIONS = "-no-emul-boot -boot-load-size 4 -boot-info-table" +MKISOFS_BOOT_OPTIONS = "-no-emul-boot -boot-load-size 4 -boot-info-table" BOOTIMG_VOLUME_ID ?= "boot" BOOTIMG_EXTRA_SPACE ?= "512" +def compute_chs(sector_z): + C = int(sector_z / (63 * 255)) + H = int((sector_z % (63 * 255)) / 63) + # convert zero-based sector to CHS format + S = int(sector_z % 63) + 1 + # munge accord to partition table format + S = (S & 0x3f) | (((C >> 8) & 0x3) << 6) + C = (C & 0xFF) + return (C, H, S) + +def mk_efi_part_table(iso, start, length): + from struct import pack + + # Compute starting and ending CHS addresses for the partition entry. + (s_C, s_H, s_S) = compute_chs(start) + (e_C, e_H, e_S) = compute_chs(start + length - 1) + + # Write the 66 byte partition table to bytes 0x1BE through 0x1FF in + # sector 0 of the .ISO. + # + # See the partition table format here: + # http://en.wikipedia.org/wiki/Master_boot_record#Sector_layout + f = open(iso, 'r+b') + f.seek(0x1BE) + f.write(pack("<8BLL48xH", 0x80, s_H, s_S, s_C, + 0xEF, e_H, e_S, e_C, start, length, 0xAA55)) + f.close() + +def install_efi_part_table(iso_img): + import subprocess + + find_efi_img_cmd = "xorriso -indev %s -find /efi.img \ + -name efi.img -exec report_lba --" % iso_img + ret = subprocess.run(find_efi_img_cmd.split(), capture_output=True) + efi_img_start = -1 + efi_img_length = -1 + for line in ret.stdout.decode(encoding='utf-8').split("\n"): + if "File data lba:" in line and "/efi.img" in line: + file_stat = line[14:].split(',') + efi_img_start = int(file_stat[1].strip()) * 4 + efi_img_length = int(int(file_stat[3].strip()) / 512) + break + if (efi_img_start < 0) or (efi_img_length < 0): + bb.fatal("Failed to determine /efi.img attributes") + mk_efi_part_table(iso_img, efi_img_start, efi_img_length) + populate_live() { - populate_kernel $1 + populate_kernel $1 if [ -s "${ROOTFS}" ]; then install -m 0644 ${ROOTFS} $1/rootfs.img fi } -build_iso() { +build_iso_base() { # Only create an ISO if we have an INITRD and the live or iso image type was selected if [ -z "${INITRD}" ] || [ "${@bb.utils.contains_any('IMAGE_FSTYPES', 'live iso', '1', '0', d)}" != "1" ]; then bbnote "ISO image will not be created." @@ -103,14 +157,6 @@ build_iso() { build_fat_img ${EFIIMGDIR} ${ISODIR}/efi.img fi - # EFI only - if [ "${PCBIOS}" != "1" ] && [ "${EFI}" = "1" ] ; then - # Work around bug in isohybrid where it requires isolinux.bin - # In the boot catalog, even though it is not used - mkdir -p ${ISODIR}/${ISOLINUXDIR} - install -m 0644 ${STAGING_DATADIR}/syslinux/isolinux.bin ${ISODIR}${ISOLINUXDIR} - fi - # We used to have support for zisofs; this is a relic of that mkisofs_compress_opts="-r" @@ -128,26 +174,40 @@ build_iso() { fi fi - if [ "${PCBIOS}" = "1" ] && [ "${EFI}" != "1" ] ; then - # PCBIOS only media - mkisofs -V ${BOOTIMG_VOLUME_ID} \ - -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ - -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ - $mkisofs_compress_opts \ - ${MKISOFS_OPTIONS} $mkisofs_iso_level ${ISODIR} + if [ "${PCBIOS}" = "1" ]; then + if [ "${EFI}" = "1" ]; then + # EFI+PCBIOS + mkisofs -A ${BOOTIMG_VOLUME_ID} -V ${BOOTIMG_VOLUME_ID} \ + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ + -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ + $mkisofs_compress_opts ${MKISOFS_BOOT_OPTIONS} $mkisofs_iso_level \ + -eltorito-alt-boot -eltorito-platform efi \ + -b efi.img -no-emul-boot \ + ${ISODIR} + isohybrid_args="-u" + else + # PCBIOS only + mkisofs -V ${BOOTIMG_VOLUME_ID} \ + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ + -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ + $mkisofs_compress_opts \ + ${MKISOFS_BOOT_OPTIONS} $mkisofs_iso_level ${ISODIR} + fi + + isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso else - # EFI only OR EFI+PCBIOS mkisofs -A ${BOOTIMG_VOLUME_ID} -V ${BOOTIMG_VOLUME_ID} \ - -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ - -b ${ISO_BOOTIMG} -c ${ISO_BOOTCAT} \ - $mkisofs_compress_opts ${MKISOFS_OPTIONS} $mkisofs_iso_level \ - -eltorito-alt-boot -eltorito-platform efi \ - -b efi.img -no-emul-boot \ + -o ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso \ + $mkisofs_compress_opts $mkisofs_iso_level \ ${ISODIR} - isohybrid_args="-u" fi +} - isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso +python build_iso() { + bb.build.exec_func("build_iso_base", d) + if d.getVar("PCBIOS") != "1" and d.getVar("EFI") == "1": + install_efi_part_table(d.getVar("IMGDEPLOYDIR") + "/" + \ + d.getVar("IMAGE_NAME") + ".iso") } build_fat_img() {
Since syslinux is only compatible with platforms that use x86-based CPUs, this change allows creation of bootable ISO images for other EFI-compatible platforms by replacing invocation of the isohybrid tool for those platforms with a python script that creates MBR partition table with a single entry that points to a bootable EFI image placed inside the ISO image. Signed-off-by: Andrey Popov <andrey.popov@yadro.com> --- meta/classes-recipe/image-live.bbclass | 124 ++++++++++++++++++------- 1 file changed, 92 insertions(+), 32 deletions(-)