[v2,2/3] image.bbclass/rootfs: set&unpack package-database

set the package-database of a "lower image" to unpack and build upon when
installing packages for the current image. This way a lean image will be
created, which only holds the packages that are not already present in the lower
image, that then could be used with overlayfs or systemd-sysext to extend the
"lower image" on demand; for development purposes on an RO lower image for
example.

Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com>
---
 meta/classes-recipe/image.bbclass         | 10 +++++++-
 meta/lib/oe/package_manager/deb/rootfs.py |  2 ++
 meta/lib/oe/package_manager/ipk/rootfs.py |  6 +++--
 meta/lib/oe/package_manager/rpm/rootfs.py |  7 ++++--
 meta/lib/oe/rootfs.py                     | 29 +++++++++++++++++++++++
 5 files changed, 49 insertions(+), 5 deletions(-)

On Tue, 2024-02-13 at 11:00 +0100, Johannes Schneider via lists.openembedded.org wrote:
> set the package-database of a "lower image" to unpack and build upon when
> installing packages for the current image. This way a lean image will be
> created, which only holds the packages that are not already present in the lower
> image, that then could be used with overlayfs or systemd-sysext to extend the
> "lower image" on demand; for development purposes on an RO lower image for
> example.
> 
> Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com>
> ---
>  meta/classes-recipe/image.bbclass         | 10 +++++++-
>  meta/lib/oe/package_manager/deb/rootfs.py |  2 ++
>  meta/lib/oe/package_manager/ipk/rootfs.py |  6 +++--
>  meta/lib/oe/package_manager/rpm/rootfs.py |  7 ++++--
>  meta/lib/oe/rootfs.py                     | 29 +++++++++++++++++++++++
>  5 files changed, 49 insertions(+), 5 deletions(-)
> 
> diff --git a/meta/classes-recipe/image.bbclass b/meta/classes-recipe/image.bbclass
> index c688c39f15..b4a2460187 100644
> --- a/meta/classes-recipe/image.bbclass
> +++ b/meta/classes-recipe/image.bbclass
> @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= ""
>  IMAGE_FEATURES[type] = "list"
>  IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password allow-empty-password allow-root-login serial-autologin-root post-install-logging overlayfs-etc"
>  
> +# Image layering:
> +# a "base image" would create a snapshot of the package-database after the
> +# installation of all packages into the rootfs is done. The next/other image
> +# "layered on-top" of the former would then import that database and install
> +# further packages; without reinstalling package dependencies that are already
> +# installed in the layer below.
>  # Generate snapshot of the package database?
>  IMAGE_GEN_PKGDBFS ?= "0"
> +# Package-database of the base image, upon which to build up a new image-layer
> +IMAGE_BASE_PKGDB ?= ""
>  
>  # Generate companion debugfs?
>  IMAGE_GEN_DEBUGFS ?= "0"
> @@ -134,7 +142,7 @@ def rootfs_variables(d):
>                   'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE',
>                   'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
>                   'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
> -                 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
> +                 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_BASE_PKGDB', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
>                   'IMAGE_INSTALL_DEBUGFS']
>      variables.extend(rootfs_command_variables(d))
>      variables.extend(variable_depends(d))
> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py
> index 43107c8663..71a21df09b 100644
> --- a/meta/lib/oe/package_manager/deb/rootfs.py
> +++ b/meta/lib/oe/package_manager/deb/rootfs.py
> @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs):
>  
>          execute_pre_post_process(self.d, deb_pre_process_cmds)
>  
> +        self._unpack_pkg_db_rootfs(['/var/lib/dpkg'])
> +
>          if self.progress_reporter:
>              self.progress_reporter.next_stage()
>              # Don't support incremental, so skip that
> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
> index 64d9bc7969..408faa8030 100644
> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
> @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs):
>          pkgs_to_install = self.manifest.parse_initial_manifest()
>          opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS')
>          opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS')
> +        opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
> +        opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>  
>          # update PM index files
>          self.pm.write_index()
>  
>          execute_pre_post_process(self.d, opkg_pre_process_cmds)
>  
> +        self._unpack_pkg_db_rootfs([opkg_dir])
> +
>          if self.progress_reporter:
>              self.progress_reporter.next_stage()
>              # Steps are a bit different in order, skip next
> @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs):
>          if self.progress_reporter:
>              self.progress_reporter.next_stage()
>  
> -        opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
> -        opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>          self._setup_pkg_db_rootfs([opkg_dir])
>          self._setup_dbg_rootfs([opkg_dir])
>  
> diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py b/meta/lib/oe/package_manager/rpm/rootfs.py
> index 673006c131..9986b13b5f 100644
> --- a/meta/lib/oe/package_manager/rpm/rootfs.py
> +++ b/meta/lib/oe/package_manager/rpm/rootfs.py
> @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs):
>          pkgs_to_install = self.manifest.parse_initial_manifest()
>          rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS')
>          rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS')
> +        package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']
>  
>          # update PM index files
>          self.pm.write_index()
>  
>          execute_pre_post_process(self.d, rpm_pre_process_cmds)
>  
> +        self._unpack_pkg_db_rootfs(package_paths)
> +
>          if self.progress_reporter:
>              self.progress_reporter.next_stage()
>  
> @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs):
>          if self.progress_reporter:
>              self.progress_reporter.next_stage()
>  
> -        self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
> -        self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
> +        self._setup_pkg_db_rootfs(package_paths)
> +        self._setup_dbg_rootfs(package_paths)
>  
>          execute_pre_post_process(self.d, rpm_post_process_cmds)
>  
> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> index 9268a02531..7ca574e6b1 100644
> --- a/meta/lib/oe/rootfs.py
> +++ b/meta/lib/oe/rootfs.py
> @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta):
>      def _cleanup(self):
>          pass
>  
> +    def _unpack_pkg_db_rootfs(self, package_paths):
> +        import tarfile
> +        gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or ''
> +        if gen_pkg_db_fs == '':
> +           return

Can be simplified to:

gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB')
if not gen_pkg_db_fs:
    return


> +
> +        fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' + self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz'
> +        if not fname:
> +            bb.warn("PKGDB does not exit:", fname)
> +            return

Shouldn't that be a hard error?

> +
> +        bb.note("  unpacking package database...")
> +        bb.utils.mkdirhier(self.image_rootfs + '-pkgdb')
> +        if fname.endswith("tar.gz"):
> +            tar = tarfile.open(fname, "r:gz")
> +            tar.extractall(path=self.image_rootfs + '-pkgdb')
> +            tar.close()

Didn't you hardcode it to tar.gz above? FWIW the python tarfile module
is also very slow.

What happens if I set IMAGE_FSTYPES_PKGDBFS to something other than
tar.gz?

> +
> +        bb.note("  Copying back package database...")
> +        for path in package_paths:
> +            try:
> +                bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
> +            except:
> +                pass
> +            if os.path.isdir(self.image_rootfs + '-pkgdb' + path):
> +                shutil.copytree(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path, symlinks=True, dirs_exist_ok=True)
> +            elif os.path.isfile(self.image_rootfs + '-pkgdb' + path):
> +                shutil.copyfile(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path)
> +
>      def _setup_pkg_db_rootfs(self, package_paths):
>          gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0'
>          if gen_pkg_db_fs != '1':

Hoi Richard,

and thanks for the feedback!
your other mail is still being processed, but to get already back to you on this one:

>> set the package-database of a "lower image" to unpack and build upon when
>> installing packages for the current image. This way a lean image will be
>> created, which only holds the packages that are not already present in the lower
>> image, that then could be used with overlayfs or systemd-sysext to extend the
>> "lower image" on demand; for development purposes on an RO lower image for
>> example.
>>
>> Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com>
>> ---
>>  meta/classes-recipe/image.bbclass         | 10 +++++++-
>>  meta/lib/oe/package_manager/deb/rootfs.py |  2 ++
>>  meta/lib/oe/package_manager/ipk/rootfs.py |  6 +++--
>>  meta/lib/oe/package_manager/rpm/rootfs.py |  7 ++++--
>>  meta/lib/oe/rootfs.py                     | 29 +++++++++++++++++++++++
>>  5 files changed, 49 insertions(+), 5 deletions(-)
>>
>> diff --git a/meta/classes-recipe/image.bbclass b/meta/classes-recipe/image.bbclass
>> index c688c39f15..b4a2460187 100644
>> --- a/meta/classes-recipe/image.bbclass
>> +++ b/meta/classes-recipe/image.bbclass
>> @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= ""
>>  IMAGE_FEATURES[type] = "list"
>>  IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password allow-empty-password allow-root-login serial-autologin-root post-install-logging overlayfs-etc"
>>
>> +# Image layering:
>> +# a "base image" would create a snapshot of the package-database after the
>> +# installation of all packages into the rootfs is done. The next/other image
>> +# "layered on-top" of the former would then import that database and install
>> +# further packages; without reinstalling package dependencies that are already
>> +# installed in the layer below.
>>  # Generate snapshot of the package database?
>>  IMAGE_GEN_PKGDBFS ?= "0"
>> +# Package-database of the base image, upon which to build up a new image-layer
>> +IMAGE_BASE_PKGDB ?= ""
>>
>>  # Generate companion debugfs?
>>  IMAGE_GEN_DEBUGFS ?= "0"
>> @@ -134,7 +142,7 @@ def rootfs_variables(d):
>>                   'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS', 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE',
>>                   'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
>>                   'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
>> -                 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
>> +                 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_BASE_PKGDB', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
>>                   'IMAGE_INSTALL_DEBUGFS']
>>      variables.extend(rootfs_command_variables(d))
>>      variables.extend(variable_depends(d))
>> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py
>> index 43107c8663..71a21df09b 100644
>> --- a/meta/lib/oe/package_manager/deb/rootfs.py
>> +++ b/meta/lib/oe/package_manager/deb/rootfs.py
>> @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs):
>>
>>          execute_pre_post_process(self.d, deb_pre_process_cmds)
>>
>> +        self._unpack_pkg_db_rootfs(['/var/lib/dpkg'])
>> +
>>          if self.progress_reporter:
>>              self.progress_reporter.next_stage()
>>              # Don't support incremental, so skip that
>> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
>> index 64d9bc7969..408faa8030 100644
>> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
>> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
>> @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs):
>>          pkgs_to_install = self.manifest.parse_initial_manifest()
>>          opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS')
>>          opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS')
>> +        opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
>> +        opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>>
>>          # update PM index files
>>          self.pm.write_index()
>>
>>          execute_pre_post_process(self.d, opkg_pre_process_cmds)
>>
>> +        self._unpack_pkg_db_rootfs([opkg_dir])
>> +
>>          if self.progress_reporter:
>>              self.progress_reporter.next_stage()
>>              # Steps are a bit different in order, skip next
>> @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs):
>>          if self.progress_reporter:
>>              self.progress_reporter.next_stage()
>>
>> -        opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
>> -        opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>>          self._setup_pkg_db_rootfs([opkg_dir])
>>          self._setup_dbg_rootfs([opkg_dir])
>>
>> diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py b/meta/lib/oe/package_manager/rpm/rootfs.py
>> index 673006c131..9986b13b5f 100644
>> --- a/meta/lib/oe/package_manager/rpm/rootfs.py
>> +++ b/meta/lib/oe/package_manager/rpm/rootfs.py
>> @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs):
>>          pkgs_to_install = self.manifest.parse_initial_manifest()
>>          rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS')
>>          rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS')
>> +        package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']
>>
>>          # update PM index files
>>          self.pm.write_index()
>>
>>          execute_pre_post_process(self.d, rpm_pre_process_cmds)
>>
>> +        self._unpack_pkg_db_rootfs(package_paths)
>> +
>>          if self.progress_reporter:
>>              self.progress_reporter.next_stage()
>>
>> @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs):
>>          if self.progress_reporter:
>>              self.progress_reporter.next_stage()
>>
>> -        self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
>> -        self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
>> +        self._setup_pkg_db_rootfs(package_paths)
>> +        self._setup_dbg_rootfs(package_paths)
>>
>>          execute_pre_post_process(self.d, rpm_post_process_cmds)
>>
>> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
>> index 9268a02531..7ca574e6b1 100644
>> --- a/meta/lib/oe/rootfs.py
>> +++ b/meta/lib/oe/rootfs.py
>> @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta):
>>      def _cleanup(self):
>>          pass
>>
>> +    def _unpack_pkg_db_rootfs(self, package_paths):
>> +        import tarfile
>> +        gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or ''
>> +        if gen_pkg_db_fs == '':
>> +           return
>
>Can be simplified to:
>
>gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB')
>if not gen_pkg_db_fs:
>    return
>

thanks! will be in V3

>
>> +
>> +        fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' + self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz'
>> +        if not fname:
>> +            bb.warn("PKGDB does not exit:", fname)
>> +            return
>
>Shouldn't that be a hard error?

good point, i also need to figure out a way to get a task-dependency there, so
that the lower-layer-base-image:setup_pkg_db is triggered when only
upper-layer-image is built separatly - thoughts/suggestions?

>
>> +
>> +        bb.note("  unpacking package database...")
>> +        bb.utils.mkdirhier(self.image_rootfs + '-pkgdb')
>> +        if fname.endswith("tar.gz"):
>> +            tar = tarfile.open(fname, "r:gz")
>> +            tar.extractall(path=self.image_rootfs + '-pkgdb')
>> +            tar.close()
>
>Didn't you hardcode it to tar.gz above? FWIW the python tarfile module
>is also very slow.
>
>
>What happens if I set IMAGE_FSTYPES_PKGDBFS to something other than
>tar.gz?
>

that's true, would i get away with dropping the whole "configurable FSTYPE" and
just always use tar.gz instead?
since the pack+unpack artifacts are only used internally while building the images,
therefor there is probably not much benefit in keeping it user-configurable

python tarfile: what alternatives are there?
is it ok to call the systems native tar?
looking around i see some uses of bb.compression in combination with tarfile,
but also others (spdx license packaging) that use the python tarfile

>> +
>> +        bb.note("  Copying back package database...")
>> +        for path in package_paths:
>> +            try:
>> +                bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
>> +            except:
>> +                pass
>> +            if os.path.isdir(self.image_rootfs + '-pkgdb' + path):
>> +                shutil.copytree(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path, symlinks=True, dirs_exist_ok=True)
>> +            elif os.path.isfile(self.image_rootfs + '-pkgdb' + path):
>> +                shutil.copyfile(self.image_rootfs + '-pkgdb' + path, self.image_rootfs + path)
>> +
>>      def _setup_pkg_db_rootfs(self, package_paths):
>>          gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0'
>>          if gen_pkg_db_fs != '1':
>
>