@@ -42,6 +42,9 @@ IMAGE_FEATURES ?= ""
IMAGE_FEATURES[type] = "list"
IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password allow-empty-password allow-root-login serial-autologin-root post-install-logging overlayfs-etc"
+# Generate snapshot of the package database?
+IMAGE_GEN_PKGDBFS ?= "0"
+
# Generate companion debugfs?
IMAGE_GEN_DEBUGFS ?= "0"
@@ -131,7 +134,8 @@ def rootfs_variables(d):
'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE',
'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
- 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', 'IMAGE_INSTALL_DEBUGFS']
+ 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
+ 'IMAGE_INSTALL_DEBUGFS']
variables.extend(rootfs_command_variables(d))
variables.extend(variable_depends(d))
return " ".join(variables)
@@ -337,6 +341,17 @@ python do_image_qa_setscene () {
}
addtask do_image_qa_setscene
+def setup_pkgdbfs_variables(d):
+ d.appendVar('IMAGE_ROOTFS', '-pkgdb')
+ if d.getVar('IMAGE_LINK_NAME'):
+ d.appendVar('IMAGE_LINK_NAME', '-pkgdb')
+ d.appendVar('IMAGE_NAME','-pkgdb')
+ d.setVar('IMAGE_FSTYPES', 'tar.gz')
+
+python setup_pkgdbfs () {
+ setup_pkgdbfs_variables(d)
+}
+
def setup_debugfs_variables(d):
d.appendVar('IMAGE_ROOTFS', '-dbg')
if d.getVar('IMAGE_LINK_NAME'):
@@ -381,6 +396,11 @@ python () {
alltypes = d.getVar('IMAGE_FSTYPES').split()
typedeps = {}
+ if d.getVar('IMAGE_GEN_PKGDBFS') == "1":
+ pkgdbfs_fstypes = ['tar.gz']
+ for t in pkgdbfs_fstypes:
+ alltypes.append("pkgdbfs_" + t)
+
if d.getVar('IMAGE_GEN_DEBUGFS') == "1":
debugfs_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS').split()
for t in debugfs_fstypes:
@@ -393,6 +413,10 @@ python () {
basetypes[baset]= []
if t not in basetypes[baset]:
basetypes[baset].append(t)
+ pkgdb = ""
+ if t.startswith("pkgdbfs_"):
+ t = t[8:]
+ pkgdb = "pkgdbfs_"
debug = ""
if t.startswith("debugfs_"):
t = t[8:]
@@ -401,6 +425,13 @@ python () {
vardeps.add('IMAGE_TYPEDEP:' + t)
if baset not in typedeps:
typedeps[baset] = set()
+ deps = [pkgdb + dep for dep in deps]
+ for dep in deps:
+ if dep not in alltypes:
+ alltypes.append(dep)
+ _add_type(dep)
+ basedep = _image_base_type(dep)
+ typedeps[baset].add(basedep)
deps = [debug + dep for dep in deps]
for dep in deps:
if dep not in alltypes:
@@ -419,6 +450,7 @@ python () {
maskedtypes = (d.getVar('IMAGE_TYPES_MASKED') or "").split()
maskedtypes = [dbg + t for t in maskedtypes for dbg in ("", "debugfs_")]
+ maskedtypes = [pkgdb + t for t in maskedtypes for pkgdb in ("", "pkgdbfs_")]
for t in basetypes:
vardeps = set()
@@ -430,6 +462,11 @@ python () {
continue
localdata = bb.data.createCopy(d)
+ pkgdb = ""
+ if t.startswith("pkgdbfs_"):
+ setup_pkgdbfs_variables(localdata)
+ pkgdb = "setup_pkgdbfs "
+ realt = t[8:]
debug = ""
if t.startswith("debugfs_"):
setup_debugfs_variables(localdata)
@@ -468,6 +505,8 @@ python () {
for ctype in sorted(ctypes):
if bt.endswith("." + ctype):
type = bt[0:-len(ctype) - 1]
+ if type.startswith("pkgdbfs_"):
+ type = type[8:]
if type.startswith("debugfs_"):
type = type[8:]
# Create input image first.
@@ -509,6 +548,9 @@ python () {
d.setVarFlag(task, 'fakeroot', '1')
d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
+ if pkgdb:
+ d.appendVarFlag(task, 'prefuncs', ' ' + pkgdb + ' set_image_size')
+
d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
@@ -25,6 +25,7 @@ def imagetypes_getdepends(d):
fstypes = set((d.getVar('IMAGE_FSTYPES') or "").split())
fstypes |= set((d.getVar('IMAGE_FSTYPES_DEBUGFS') or "").split())
+ fstypes |= set('tar.gz') # hardcoded for 'pkgdbfs'
deprecated = set()
deps = set()
@@ -215,6 +215,7 @@ IMAGE_FEATURES[doc] = "The primary list of features to include in an image. Conf
IMAGE_FSTYPES[doc] = "Formats of root filesystem images that you want to have created."
IMAGE_FSTYPES_DEBUGFS[doc] = "Formats of the debug root filesystem images that you want to have created."
IMAGE_GEN_DEBUGFS[doc] = "When set to '1', generate a companion debug object/source filesystem image."
+IMAGE_GEN_PKGDBFS[doc] = "When set to '1', create a snapshot of the package-manager state after the rootfs has been assembled."
IMAGE_INSTALL[doc] = "Specifies the packages to install into an image. Image recipes set IMAGE_INSTALL to specify the packages to install into an image through image.bbclass."
IMAGE_LINGUAS[doc] = "Specifies the list of locales to install into the image during the root filesystem construction process."
IMAGE_NAME[doc] = "The name of the output image files minus the extension."
@@ -178,6 +178,7 @@ class PkgRootfs(DpkgOpkgRootfs):
if self.progress_reporter:
self.progress_reporter.next_stage()
+ self._setup_pkg_db_rootfs(['/var/lib/dpkg'])
self._setup_dbg_rootfs(['/var/lib/dpkg'])
self.pm.fix_broken_dependencies()
@@ -319,6 +319,7 @@ class PkgRootfs(DpkgOpkgRootfs):
opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
+ self._setup_pkg_db_rootfs([opkg_dir])
self._setup_dbg_rootfs([opkg_dir])
execute_pre_post_process(self.d, opkg_post_process_cmds)
@@ -110,6 +110,7 @@ class PkgRootfs(Rootfs):
if self.progress_reporter:
self.progress_reporter.next_stage()
+ self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
execute_pre_post_process(self.d, rpm_post_process_cmds)
@@ -106,6 +106,26 @@ class Rootfs(object, metaclass=ABCMeta):
def _cleanup(self):
pass
+ def _setup_pkg_db_rootfs(self, package_paths):
+ gen_pkg_db_fs = bb.utils.to_boolean(self.d.getVar('IMAGE_GEN_PKGDBFS'))
+ if not gen_pkg_db_fs:
+ return
+
+ bb.note(" Creating pkg-db rootfs...")
+ try:
+ shutil.rmtree(self.image_rootfs + '-pkgdb')
+ except (FileNotFoundError, NotADirectoryError):
+ pass
+ bb.utils.mkdirhier(self.image_rootfs + "-pkgdb")
+
+ bb.note(" Copying back package database...")
+ for path in package_paths:
+ bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
+ if os.path.isdir(self.image_rootfs + path):
+ shutil.copytree(self.image_rootfs + path, self.image_rootfs + '-pkgdb' + path, symlinks=True)
+ elif os.path.isfile(self.image_rootfs + path):
+ shutil.copyfile(self.image_rootfs + path, self.image_rootfs + '-pkgdb' + path)
+
def _setup_dbg_rootfs(self, package_paths):
gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
if gen_debugfs != '1':
@@ -302,6 +302,53 @@ SKIP_RECIPE[busybox] = "Don't build this"
result = runCmd('objdump --syms %s | grep debug' % t)
self.assertTrue("debug" in result.output, msg='Failed to find debug symbol: %s' % result.output)
+ def test_image_gen_pkgdbfs(self):
+ """
+ Summary: Check pkgdb snapshot
+ Expected: 1. core-image-minimal can be build with IMAGE_GEN_PKGDBFS variable set
+ 2. *rootfs-pkgdb.tar.gz snapshot of the package manager state is created,
+ after the rootfs of the core-image-minimal is put together
+ 3. check that the restored package index is usable by the default package-manager
+ PACKAGE_CLASS=package_ipk -> opkg in this case.
+ """
+
+ image = 'core-image-minimal'
+ features = 'IMAGE_GEN_PKGDBFS = "1"\n'
+ self.write_config(features)
+
+ bitbake(image)
+
+ img_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME', 'MACHINE', 'DEFAULTTUNE'], image)
+
+ tar_file = os.path.join(img_vars['DEPLOY_DIR_IMAGE'], "%s-pkgdb.tar.gz" % (img_vars['IMAGE_LINK_NAME']))
+ self.assertTrue(os.path.exists(tar_file), 'pkgdb filesystem not generated at %s' % tar_file)
+ result = runCmd('cd %s; tar xvf %s' % (img_vars['DEPLOY_DIR_IMAGE'], tar_file))
+ self.assertEqual(result.status, 0, msg='Failed to extract %s: %s' % (tar_file, result.output))
+ self.assertTrue(os.path.exists(os.path.join(img_vars['DEPLOY_DIR_IMAGE'], 'var/lib/opkg/status')), 'opkg\'s status file was not present in: %s' % tar_file)
+
+ bitbake("opkg-native")
+ bitbake("build-sysroots -c build_native_sysroot")
+
+ native_vars = get_bb_vars(["STAGING_DIR", "BUILD_ARCH", "bindir_native", "libdir_native"])
+ stagingbindir = os.path.join(native_vars["STAGING_DIR"], native_vars["BUILD_ARCH"], native_vars["bindir_native"].lstrip(os.sep))
+
+ # some 'add-arch' have '-' replaced by '_', some not; adding both does not hurt
+ cmd = os.path.join(stagingbindir, 'opkg') + ' --volatile-cache --offline-root=%s --add-arch %s:11 --add-arch %s:12 list-installed' % (img_vars['DEPLOY_DIR_IMAGE'], img_vars["MACHINE"], img_vars["MACHINE"].replace('-','_'))
+ self.logger.debug('running cmd: %s' % cmd)
+ result = runCmd(cmd)
+ self.logger.debug("list-installed:\n%s" % result.output)
+ self.assertEqual(result.status, 0, msg='Failed to run package manager on unpacked pkgdb %s: %s' % (tar_file, result.output))
+ # check for a 'random, very architecture specific' package
+ self.assertTrue("kernel-image" in result.output, msg='Failed query installed packages')
+
+ cmd = os.path.join(stagingbindir, 'opkg') + ' --volatile-cache --offline-root=%s --add-arch %s:11 --add-arch %s:12 list-installed' % (img_vars['DEPLOY_DIR_IMAGE'], img_vars["DEFAULTTUNE"], img_vars["DEFAULTTUNE"].replace('-','_'))
+ self.logger.debug('running cmd: %s' % cmd)
+ result = runCmd(cmd)
+ self.logger.debug("list-installed:\n%s" % result.output)
+ self.assertEqual(result.status, 0, msg='Failed to run package manager on unpacked pkgdb %s: %s' % (tar_file, result.output))
+ # check for a common base package, a random library for example:
+ self.assertTrue("libc6" in result.output, msg='Failed query installed packages')
+
def test_empty_image(self):
"""Test creation of image with no packages"""
image = 'test-empty-image'
archive the package database after the rootfs has been put together as *rootfs-pkdbfs.tar.gz, and put it into the deploy folder. This creates a snapshot of the package mangers state at the point in time when all dependencies have been resolved and installed; which could be used by "extension images" to built upon. Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com> --- meta/classes-recipe/image.bbclass | 44 ++++++++++++++++- meta/classes-recipe/image_types.bbclass | 1 + meta/conf/documentation.conf | 1 + meta/lib/oe/package_manager/deb/rootfs.py | 1 + meta/lib/oe/package_manager/ipk/rootfs.py | 1 + meta/lib/oe/package_manager/rpm/rootfs.py | 1 + meta/lib/oe/rootfs.py | 20 ++++++++ meta/lib/oeqa/selftest/cases/imagefeatures.py | 47 +++++++++++++++++++ 8 files changed, 115 insertions(+), 1 deletion(-)