Message ID | 20240213100012.252237-3-johannes.schneider@leica-geosystems.com |
---|---|
State | New |
Headers | show |
Series | pkg-database and systemd-sysext image | expand |
On Tue, 2024-02-13 at 11:00 +0100, Johannes Schneider via lists.openembedded.org wrote: > set the package-database of a "lower image" to unpack and build upon when > installing packages for the current image. This way a lean image will be > created, which only holds the packages that are not already present in the lower > image, that then could be used with overlayfs or systemd-sysext to extend the > "lower image" on demand; for development purposes on an RO lower image for > example. > > Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com> > --- > meta/classes-recipe/image.bbclass | 10 +++++++- > meta/lib/oe/package_manager/deb/rootfs.py | 2 ++ > meta/lib/oe/package_manager/ipk/rootfs.py | 6 +++-- > meta/lib/oe/package_manager/rpm/rootfs.py | 7 ++++-- > meta/lib/oe/rootfs.py | 29 +++++++++++++++++++++++ > 5 files changed, 49 insertions(+), 5 deletions(-) > > diff --git a/meta/classes-recipe/image.bbclass b/meta/classes-recipe/image.bbclass > index c688c39f15..b4a2460187 100644 > --- a/meta/classes-recipe/image.bbclass > +++ b/meta/classes-recipe/image.bbclass > @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= "" > IMAGE_FEATURES[type] = "list" > IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password allow-empty-password allow-root-login serial-autologin-root post-install-logging overlayfs-etc" > > +# Image layering: > +# a "base image" would create a snapshot of the package-database after the > +# installation of all packages into the rootfs is done. The next/other image > +# "layered on-top" of the former would then import that database and install > +# further packages; without reinstalling package dependencies that are already > +# installed in the layer below. > # Generate snapshot of the package database? > IMAGE_GEN_PKGDBFS ?= "0" > +# Package-database of the base image, upon which to build up a new image-layer > +IMAGE_BASE_PKGDB ?= "" > > # Generate companion debugfs? > IMAGE_GEN_DEBUGFS ?= "0" > @@ -134,7 +142,7 @@ def rootfs_variables(d): > 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE', > 'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS', > 'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', > - 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', > + 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_BASE_PKGDB', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', > 'IMAGE_INSTALL_DEBUGFS'] > variables.extend(rootfs_command_variables(d)) > variables.extend(variable_depends(d)) > diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py > index 43107c8663..71a21df09b 100644 > --- a/meta/lib/oe/package_manager/deb/rootfs.py > +++ b/meta/lib/oe/package_manager/deb/rootfs.py > @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs): > > execute_pre_post_process(self.d, deb_pre_process_cmds) > > + self._unpack_pkg_db_rootfs(['/var/lib/dpkg']) > + > if self.progress_reporter: > self.progress_reporter.next_stage() > # Don't support incremental, so skip that > diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py > index 64d9bc7969..408faa8030 100644 > --- a/meta/lib/oe/package_manager/ipk/rootfs.py > +++ b/meta/lib/oe/package_manager/ipk/rootfs.py > @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs): > pkgs_to_install = self.manifest.parse_initial_manifest() > opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') > opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') > + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') > + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') > > # update PM index files > self.pm.write_index() > > execute_pre_post_process(self.d, opkg_pre_process_cmds) > > + self._unpack_pkg_db_rootfs([opkg_dir]) > + > if self.progress_reporter: > self.progress_reporter.next_stage() > # Steps are a bit different in order, skip next > @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs): > if self.progress_reporter: > self.progress_reporter.next_stage() > > - opkg_lib_dir = self.d.getVar('OPKGLIBDIR') > - opkg_dir = os.path.join(opkg_lib_dir, 'opkg') > self._setup_pkg_db_rootfs([opkg_dir]) > self._setup_dbg_rootfs([opkg_dir]) > > diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py b/meta/lib/oe/package_manager/rpm/rootfs.py > index 673006c131..9986b13b5f 100644 > --- a/meta/lib/oe/package_manager/rpm/rootfs.py > +++ b/meta/lib/oe/package_manager/rpm/rootfs.py > @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs): > pkgs_to_install = self.manifest.parse_initial_manifest() > rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') > rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') > + package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'] > > # update PM index files > self.pm.write_index() > > execute_pre_post_process(self.d, rpm_pre_process_cmds) > > + self._unpack_pkg_db_rootfs(package_paths) > + > if self.progress_reporter: > self.progress_reporter.next_stage() > > @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs): > if self.progress_reporter: > self.progress_reporter.next_stage() > > - self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) > - self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) > + self._setup_pkg_db_rootfs(package_paths) > + self._setup_dbg_rootfs(package_paths) > > execute_pre_post_process(self.d, rpm_post_process_cmds) > > diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py > index 9268a02531..7ca574e6b1 100644 > --- a/meta/lib/oe/rootfs.py > +++ b/meta/lib/oe/rootfs.py > @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta): > def _cleanup(self): > pass > > + def _unpack_pkg_db_rootfs(self, package_paths): > + import tarfile > + gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or '' > + if gen_pkg_db_fs == '': > + return Can be simplified to: gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') if not gen_pkg_db_fs: return > + > + fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' + self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz' > + if not fname: > + bb.warn("PKGDB does not exit:", fname) > + return Shouldn't that be a hard error? > + > + bb.note(" unpacking package database...") > + bb.utils.mkdirhier(self.image_rootfs + '-pkgdb') > + if fname.endswith("tar.gz"): > + tar = tarfile.open(fname, "r:gz") > + tar.extractall(path=self.image_rootfs + '-pkgdb') > + tar.close() Didn't you hardcode it to tar.gz above? FWIW the python tarfile module is also very slow. What happens if I set IMAGE_FSTYPES_PKGDBFS to something other than tar.gz? > + > + bb.note(" Copying back package database...") > + for path in package_paths: > + try: > + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path)) > + except: > + pass > + if os.path.isdir(self.image_rootfs + '-pkgdb' + path): > + shutil.copytree(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path, symlinks=True, dirs_exist_ok=True) > + elif os.path.isfile(self.image_rootfs + '-pkgdb' + path): > + shutil.copyfile(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path) > + > def _setup_pkg_db_rootfs(self, package_paths): > gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0' > if gen_pkg_db_fs != '1':
Hoi Richard, and thanks for the feedback! your other mail is still being processed, but to get already back to you on this one: >> set the package-database of a "lower image" to unpack and build upon when >> installing packages for the current image. This way a lean image will be >> created, which only holds the packages that are not already present in the lower >> image, that then could be used with overlayfs or systemd-sysext to extend the >> "lower image" on demand; for development purposes on an RO lower image for >> example. >> >> Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com> >> --- >> meta/classes-recipe/image.bbclass | 10 +++++++- >> meta/lib/oe/package_manager/deb/rootfs.py | 2 ++ >> meta/lib/oe/package_manager/ipk/rootfs.py | 6 +++-- >> meta/lib/oe/package_manager/rpm/rootfs.py | 7 ++++-- >> meta/lib/oe/rootfs.py | 29 +++++++++++++++++++++++ >> 5 files changed, 49 insertions(+), 5 deletions(-) >> >> diff --git a/meta/classes-recipe/image.bbclass b/meta/classes-recipe/image.bbclass >> index c688c39f15..b4a2460187 100644 >> --- a/meta/classes-recipe/image.bbclass >> +++ b/meta/classes-recipe/image.bbclass >> @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= "" >> IMAGE_FEATURES[type] = "list" >> IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password allow-empty-password allow-root-login serial-autologin-root post-install-logging overlayfs-etc" >> >> +# Image layering: >> +# a "base image" would create a snapshot of the package-database after the >> +# installation of all packages into the rootfs is done. The next/other image >> +# "layered on-top" of the former would then import that database and install >> +# further packages; without reinstalling package dependencies that are already >> +# installed in the layer below. >> # Generate snapshot of the package database? >> IMAGE_GEN_PKGDBFS ?= "0" >> +# Package-database of the base image, upon which to build up a new image-layer >> +IMAGE_BASE_PKGDB ?= "" >> >> # Generate companion debugfs? >> IMAGE_GEN_DEBUGFS ?= "0" >> @@ -134,7 +142,7 @@ def rootfs_variables(d): >> 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE', >> 'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS', >> 'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', >> - 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', >> + 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_BASE_PKGDB', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', >> 'IMAGE_INSTALL_DEBUGFS'] >> variables.extend(rootfs_command_variables(d)) >> variables.extend(variable_depends(d)) >> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py >> index 43107c8663..71a21df09b 100644 >> --- a/meta/lib/oe/package_manager/deb/rootfs.py >> +++ b/meta/lib/oe/package_manager/deb/rootfs.py >> @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs): >> >> execute_pre_post_process(self.d, deb_pre_process_cmds) >> >> + self._unpack_pkg_db_rootfs(['/var/lib/dpkg']) >> + >> if self.progress_reporter: >> self.progress_reporter.next_stage() >> # Don't support incremental, so skip that >> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py >> index 64d9bc7969..408faa8030 100644 >> --- a/meta/lib/oe/package_manager/ipk/rootfs.py >> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py >> @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs): >> pkgs_to_install = self.manifest.parse_initial_manifest() >> opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') >> opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') >> + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') >> + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') >> >> # update PM index files >> self.pm.write_index() >> >> execute_pre_post_process(self.d, opkg_pre_process_cmds) >> >> + self._unpack_pkg_db_rootfs([opkg_dir]) >> + >> if self.progress_reporter: >> self.progress_reporter.next_stage() >> # Steps are a bit different in order, skip next >> @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs): >> if self.progress_reporter: >> self.progress_reporter.next_stage() >> >> - opkg_lib_dir = self.d.getVar('OPKGLIBDIR') >> - opkg_dir = os.path.join(opkg_lib_dir, 'opkg') >> self._setup_pkg_db_rootfs([opkg_dir]) >> self._setup_dbg_rootfs([opkg_dir]) >> >> diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py b/meta/lib/oe/package_manager/rpm/rootfs.py >> index 673006c131..9986b13b5f 100644 >> --- a/meta/lib/oe/package_manager/rpm/rootfs.py >> +++ b/meta/lib/oe/package_manager/rpm/rootfs.py >> @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs): >> pkgs_to_install = self.manifest.parse_initial_manifest() >> rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') >> rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') >> + package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'] >> >> # update PM index files >> self.pm.write_index() >> >> execute_pre_post_process(self.d, rpm_pre_process_cmds) >> >> + self._unpack_pkg_db_rootfs(package_paths) >> + >> if self.progress_reporter: >> self.progress_reporter.next_stage() >> >> @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs): >> if self.progress_reporter: >> self.progress_reporter.next_stage() >> >> - self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) >> - self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) >> + self._setup_pkg_db_rootfs(package_paths) >> + self._setup_dbg_rootfs(package_paths) >> >> execute_pre_post_process(self.d, rpm_post_process_cmds) >> >> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py >> index 9268a02531..7ca574e6b1 100644 >> --- a/meta/lib/oe/rootfs.py >> +++ b/meta/lib/oe/rootfs.py >> @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta): >> def _cleanup(self): >> pass >> >> + def _unpack_pkg_db_rootfs(self, package_paths): >> + import tarfile >> + gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or '' >> + if gen_pkg_db_fs == '': >> + return > >Can be simplified to: > >gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') >if not gen_pkg_db_fs: > return > thanks! will be in V3 > >> + >> + fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' + self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz' >> + if not fname: >> + bb.warn("PKGDB does not exit:", fname) >> + return > >Shouldn't that be a hard error? good point, i also need to figure out a way to get a task-dependency there, so that the lower-layer-base-image:setup_pkg_db is triggered when only upper-layer-image is built separatly - thoughts/suggestions? > >> + >> + bb.note(" unpacking package database...") >> + bb.utils.mkdirhier(self.image_rootfs + '-pkgdb') >> + if fname.endswith("tar.gz"): >> + tar = tarfile.open(fname, "r:gz") >> + tar.extractall(path=self.image_rootfs + '-pkgdb') >> + tar.close() > >Didn't you hardcode it to tar.gz above? FWIW the python tarfile module >is also very slow. > > >What happens if I set IMAGE_FSTYPES_PKGDBFS to something other than >tar.gz? > that's true, would i get away with dropping the whole "configurable FSTYPE" and just always use tar.gz instead? since the pack+unpack artifacts are only used internally while building the images, therefor there is probably not much benefit in keeping it user-configurable python tarfile: what alternatives are there? is it ok to call the systems native tar? looking around i see some uses of bb.compression in combination with tarfile, but also others (spdx license packaging) that use the python tarfile >> + >> + bb.note(" Copying back package database...") >> + for path in package_paths: >> + try: >> + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path)) >> + except: >> + pass >> + if os.path.isdir(self.image_rootfs + '-pkgdb' + path): >> + shutil.copytree(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path, symlinks=True, dirs_exist_ok=True) >> + elif os.path.isfile(self.image_rootfs + '-pkgdb' + path): >> + shutil.copyfile(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path) >> + >> def _setup_pkg_db_rootfs(self, package_paths): >> gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0' >> if gen_pkg_db_fs != '1': > >
diff --git a/meta/classes-recipe/image.bbclass b/meta/classes-recipe/image.bbclass index c688c39f15..b4a2460187 100644 --- a/meta/classes-recipe/image.bbclass +++ b/meta/classes-recipe/image.bbclass @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= "" IMAGE_FEATURES[type] = "list" IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password allow-empty-password allow-root-login serial-autologin-root post-install-logging overlayfs-etc" +# Image layering: +# a "base image" would create a snapshot of the package-database after the +# installation of all packages into the rootfs is done. The next/other image +# "layered on-top" of the former would then import that database and install +# further packages; without reinstalling package dependencies that are already +# installed in the layer below. # Generate snapshot of the package database? IMAGE_GEN_PKGDBFS ?= "0" +# Package-database of the base image, upon which to build up a new image-layer +IMAGE_BASE_PKGDB ?= "" # Generate companion debugfs? IMAGE_GEN_DEBUGFS ?= "0" @@ -134,7 +142,7 @@ def rootfs_variables(d): 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE', 'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS', 'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS', - 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', + 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_BASE_PKGDB', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', 'IMAGE_INSTALL_DEBUGFS'] variables.extend(rootfs_command_variables(d)) variables.extend(variable_depends(d)) diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py index 43107c8663..71a21df09b 100644 --- a/meta/lib/oe/package_manager/deb/rootfs.py +++ b/meta/lib/oe/package_manager/deb/rootfs.py @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs): execute_pre_post_process(self.d, deb_pre_process_cmds) + self._unpack_pkg_db_rootfs(['/var/lib/dpkg']) + if self.progress_reporter: self.progress_reporter.next_stage() # Don't support incremental, so skip that diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py index 64d9bc7969..408faa8030 100644 --- a/meta/lib/oe/package_manager/ipk/rootfs.py +++ b/meta/lib/oe/package_manager/ipk/rootfs.py @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs): pkgs_to_install = self.manifest.parse_initial_manifest() opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS') opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS') + opkg_lib_dir = self.d.getVar('OPKGLIBDIR') + opkg_dir = os.path.join(opkg_lib_dir, 'opkg') # update PM index files self.pm.write_index() execute_pre_post_process(self.d, opkg_pre_process_cmds) + self._unpack_pkg_db_rootfs([opkg_dir]) + if self.progress_reporter: self.progress_reporter.next_stage() # Steps are a bit different in order, skip next @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs): if self.progress_reporter: self.progress_reporter.next_stage() - opkg_lib_dir = self.d.getVar('OPKGLIBDIR') - opkg_dir = os.path.join(opkg_lib_dir, 'opkg') self._setup_pkg_db_rootfs([opkg_dir]) self._setup_dbg_rootfs([opkg_dir]) diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py b/meta/lib/oe/package_manager/rpm/rootfs.py index 673006c131..9986b13b5f 100644 --- a/meta/lib/oe/package_manager/rpm/rootfs.py +++ b/meta/lib/oe/package_manager/rpm/rootfs.py @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs): pkgs_to_install = self.manifest.parse_initial_manifest() rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS') + package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'] # update PM index files self.pm.write_index() execute_pre_post_process(self.d, rpm_pre_process_cmds) + self._unpack_pkg_db_rootfs(package_paths) + if self.progress_reporter: self.progress_reporter.next_stage() @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs): if self.progress_reporter: self.progress_reporter.next_stage() - self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) - self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']) + self._setup_pkg_db_rootfs(package_paths) + self._setup_dbg_rootfs(package_paths) execute_pre_post_process(self.d, rpm_post_process_cmds) diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py index 9268a02531..7ca574e6b1 100644 --- a/meta/lib/oe/rootfs.py +++ b/meta/lib/oe/rootfs.py @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta): def _cleanup(self): pass + def _unpack_pkg_db_rootfs(self, package_paths): + import tarfile + gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or '' + if gen_pkg_db_fs == '': + return + + fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' + self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz' + if not fname: + bb.warn("PKGDB does not exit:", fname) + return + + bb.note(" unpacking package database...") + bb.utils.mkdirhier(self.image_rootfs + '-pkgdb') + if fname.endswith("tar.gz"): + tar = tarfile.open(fname, "r:gz") + tar.extractall(path=self.image_rootfs + '-pkgdb') + tar.close() + + bb.note(" Copying back package database...") + for path in package_paths: + try: + bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path)) + except: + pass + if os.path.isdir(self.image_rootfs + '-pkgdb' + path): + shutil.copytree(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path, symlinks=True, dirs_exist_ok=True) + elif os.path.isfile(self.image_rootfs + '-pkgdb' + path): + shutil.copyfile(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path) + def _setup_pkg_db_rootfs(self, package_paths): gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0' if gen_pkg_db_fs != '1':
set the package-database of a "lower image" to unpack and build upon when installing packages for the current image. This way a lean image will be created, which only holds the packages that are not already present in the lower image, that then could be used with overlayfs or systemd-sysext to extend the "lower image" on demand; for development purposes on an RO lower image for example. Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com> --- meta/classes-recipe/image.bbclass | 10 +++++++- meta/lib/oe/package_manager/deb/rootfs.py | 2 ++ meta/lib/oe/package_manager/ipk/rootfs.py | 6 +++-- meta/lib/oe/package_manager/rpm/rootfs.py | 7 ++++-- meta/lib/oe/rootfs.py | 29 +++++++++++++++++++++++ 5 files changed, 49 insertions(+), 5 deletions(-)