Patchwork [1/5] package_manager.py: support ipk incremental image generation

login
register
mail settings
Submitter Hongxu Jia
Date Feb. 18, 2014, 9:42 a.m.
Message ID <e35ba30b31bc2579f237c58c4d991c3f90657a1d.1392716065.git.hongxu.jia@windriver.com>
Download mbox | patch
Permalink /patch/66897/
State New
Headers show

Comments

Hongxu Jia - Feb. 18, 2014, 9:42 a.m.
While incremental image generation enabled, 'load_old_install_solution'
is used to determine what we've got in the previous (existed) image and
'dump_install_solution' to determine what we need to install in the
current image.

The 'backup_packaging_data' is used to back up the current opkg database.

The 'recovery_packaging_data' is used to recover the opkg database which
backed up by the previous image creation.

Tweak 'remove' function in OpkgPM class, which the options for remove
with dependencies was incorrect.

[YOCTO #1894]

Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
---
 meta/lib/oe/package_manager.py | 106 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 102 insertions(+), 4 deletions(-)
Laurentiu Palcu - Feb. 18, 2014, 2:38 p.m.
On Tue, Feb 18, 2014 at 05:42:24PM +0800, Hongxu Jia wrote:
> While incremental image generation enabled, 'load_old_install_solution'
> is used to determine what we've got in the previous (existed) image and
> 'dump_install_solution' to determine what we need to install in the
> current image.
> 
> The 'backup_packaging_data' is used to back up the current opkg database.
> 
> The 'recovery_packaging_data' is used to recover the opkg database which
> backed up by the previous image creation.
> 
> Tweak 'remove' function in OpkgPM class, which the options for remove
> with dependencies was incorrect.
> 
> [YOCTO #1894]
> 
> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
> ---
>  meta/lib/oe/package_manager.py | 106 +++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 102 insertions(+), 4 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index 6dc8fbd..4ea9677 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -934,12 +934,13 @@ class RpmPM(PackageManager):
>  
>  
>  class OpkgPM(PackageManager):
> -    def __init__(self, d, target_rootfs, config_file, archs):
> +    def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
>          super(OpkgPM, self).__init__(d)
>  
>          self.target_rootfs = target_rootfs
>          self.config_file = config_file
>          self.pkg_archs = archs
> +        self.task_name = task_name
>  
>          self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
>          self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
> @@ -956,6 +957,13 @@ class OpkgPM(PackageManager):
>  
>          bb.utils.mkdirhier(self.opkg_dir)
>  
> +        self.solution_manifest = self.d.expand('${T}/saved/%s_solution' %
> +                                               self.task_name)
> +        self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
> +        if not os.path.exists(self.d.expand('${T}/saved')):
> +            bb.utils.mkdirhier(self.d.expand('${T}/saved'))
> +
> +
>          if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
>              self._create_config()
>          else:
> @@ -1075,7 +1083,9 @@ class OpkgPM(PackageManager):
>  
>          try:
>              bb.note("Installing the following packages: %s" % ' '.join(pkgs))
> -            subprocess.check_output(cmd.split())
> +            bb.note(cmd)
> +            output = subprocess.check_output(cmd.split())
> +            bb.note(output)
>          except subprocess.CalledProcessError as e:
>              (bb.fatal, bb.note)[attempt_only]("Unable to install packages. "
>                                                "Command '%s' returned %d:\n%s" %
> @@ -1083,14 +1093,16 @@ class OpkgPM(PackageManager):
>  
>      def remove(self, pkgs, with_dependencies=True):
>          if with_dependencies:
> -            cmd = "%s %s remove %s" % \
> +            cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
>                  (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
>          else:
>              cmd = "%s %s --force-depends remove %s" % \
>                  (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
>  
>          try:
> -            subprocess.check_output(cmd.split())
> +            bb.note(cmd)
> +            output = subprocess.check_output(cmd.split())
> +            bb.note(output)
>          except subprocess.CalledProcessError as e:
>              bb.fatal("Unable to remove packages. Command '%s' "
>                       "returned %d:\n%s" % (e.cmd, e.returncode, e.output))
> @@ -1175,6 +1187,92 @@ class OpkgPM(PackageManager):
>                      else:
>                          status.write(line + "\n")
>  
> +    '''
> +    If incremental install, we need to determine what we've got,
> +    what we need to add, and what to remove...
> +    The dump_install_solution will dump and save the new install
> +    solution.
> +    '''
> +    def dump_install_solution(self, pkgs):
Why not have this function in the Manifest class? We have an API in
place: Manifest.create_final() that should probably take care of
anything related to manifest(s) creation after all packages are installed.

> +        bb.note('creating new install solution for incremental install')
> +        if len(pkgs) == 0:
> +            return
> +
> +        install_pkgs = list()
> +
> +        # Create an temp dir as opkg root for simulating the installation
> +        temp_rootfs = self.d.expand('${T}/opkg')
> +        temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg')
> +        bb.utils.mkdirhier(temp_opkg_dir)
> +
> +        opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
> +        opkg_args += self.d.getVar("OPKG_ARGS", True)
> +
> +        cmd = "%s %s update" % (self.opkg_cmd,
> +                                opkg_args)
> +        try:
> +            subprocess.check_output(cmd, shell=True)
> +        except subprocess.CalledProcessError as e:
> +            bb.note("Unable to dump install packages. Command '%s' "
> +                    "returned %d:\n%s" % (cmd, e.returncode, e.output))
> +
> +        # Simulate installation from zero
> +        cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
> +                                                opkg_args,
> +                                                ' '.join(pkgs))
> +        cmd += "| awk '/^Installing/{print $2}' "
> +        cmd += "| sort -u -o %s" % self.solution_manifest
Do we need to preserve this bash one-liner? Why not handle the command
output using Python?

> +        try:
> +            subprocess.check_output(cmd, shell=True)
> +            with open(self.solution_manifest, 'r') as manifest:
> +                for pkg in manifest.read().split('\n'):
> +                    install_pkgs.append(pkg)
> +        except subprocess.CalledProcessError as e:
> +            bb.note("Unable to dump install packages. Command '%s' "
> +                    "returned %d:\n%s" % (cmd, e.returncode, e.output))
> +
> +        bb.utils.remove(temp_rootfs, True)
> +
> +        return install_pkgs
> +
> +    '''
> +    If incremental install, we need to determine what we've got,
> +    what we need to add, and what to remove...
> +    The load_old_install_solution will load the previous install
> +    solution
> +    '''
> +    def load_old_install_solution(self):
Same here: why not put it in the Manifest class? As we already have the
Manifest.parse_initial_manifest(), we can have something similar for the
final_manifest (or install_solution, etc) parsing.

> +        bb.note('load old install solution for incremental install')
> +        installed_pkgs = list()
> +        if not os.path.exists(self.solution_manifest):
> +            bb.note('old install solution not exist')
> +            return installed_pkgs
> +
> +        with open(self.solution_manifest, 'r') as manifest:
> +            for pkg in manifest.read().split('\n'):
> +                installed_pkgs.append(pkg.strip())
> +
> +        return installed_pkgs
> +
> +    def backup_packaging_data(self):
> +        # Save the opkglib for increment rpm image generation
s/rpm/opkg/

> +        if os.path.exists(self.saved_opkg_dir):
> +            bb.utils.remove(self.saved_opkg_dir, True)
> +        shutil.copytree(self.opkg_dir,
> +                        self.saved_opkg_dir,
> +                        symlinks=True)
> +
> +    def recovery_packaging_data(self):
s/recovery/recover/ ?

> +        # Move the opkglib back
> +        if os.path.exists(self.saved_opkg_dir):
> +            if os.path.exists(self.opkg_dir):
> +                bb.utils.remove(self.opkg_dir, True)
> +
> +            bb.note('Recovery packaging data')
> +            shutil.copytree(self.saved_opkg_dir,
> +                            self.opkg_dir,
> +                            symlinks=True)
> +
>  
>  class DpkgPM(PackageManager):
>      def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
> -- 
> 1.8.1.2
>
Hongxu Jia - Feb. 19, 2014, 9:39 a.m.
On 02/18/2014 10:38 PM, Laurentiu Palcu wrote:
> On Tue, Feb 18, 2014 at 05:42:24PM +0800, Hongxu Jia wrote:
>> While incremental image generation enabled, 'load_old_install_solution'
>> is used to determine what we've got in the previous (existed) image and
>> 'dump_install_solution' to determine what we need to install in the
>> current image.
>>
>> The 'backup_packaging_data' is used to back up the current opkg database.
>>
>> The 'recovery_packaging_data' is used to recover the opkg database which
>> backed up by the previous image creation.
>>
>> Tweak 'remove' function in OpkgPM class, which the options for remove
>> with dependencies was incorrect.
>>
>> [YOCTO #1894]
>>
>> Signed-off-by: Hongxu Jia <hongxu.jia@windriver.com>
>> ---
>>   meta/lib/oe/package_manager.py | 106 +++++++++++++++++++++++++++++++++++++++--
>>   1 file changed, 102 insertions(+), 4 deletions(-)
>>
>> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
>> index 6dc8fbd..4ea9677 100644
>> --- a/meta/lib/oe/package_manager.py
>> +++ b/meta/lib/oe/package_manager.py
>> @@ -934,12 +934,13 @@ class RpmPM(PackageManager):
>>   
>>   
>>   class OpkgPM(PackageManager):
>> -    def __init__(self, d, target_rootfs, config_file, archs):
>> +    def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
>>           super(OpkgPM, self).__init__(d)
>>   
>>           self.target_rootfs = target_rootfs
>>           self.config_file = config_file
>>           self.pkg_archs = archs
>> +        self.task_name = task_name
>>   
>>           self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
>>           self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
>> @@ -956,6 +957,13 @@ class OpkgPM(PackageManager):
>>   
>>           bb.utils.mkdirhier(self.opkg_dir)
>>   
>> +        self.solution_manifest = self.d.expand('${T}/saved/%s_solution' %
>> +                                               self.task_name)
>> +        self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
>> +        if not os.path.exists(self.d.expand('${T}/saved')):
>> +            bb.utils.mkdirhier(self.d.expand('${T}/saved'))
>> +
>> +
>>           if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
>>               self._create_config()
>>           else:
>> @@ -1075,7 +1083,9 @@ class OpkgPM(PackageManager):
>>   
>>           try:
>>               bb.note("Installing the following packages: %s" % ' '.join(pkgs))
>> -            subprocess.check_output(cmd.split())
>> +            bb.note(cmd)
>> +            output = subprocess.check_output(cmd.split())
>> +            bb.note(output)
>>           except subprocess.CalledProcessError as e:
>>               (bb.fatal, bb.note)[attempt_only]("Unable to install packages. "
>>                                                 "Command '%s' returned %d:\n%s" %
>> @@ -1083,14 +1093,16 @@ class OpkgPM(PackageManager):
>>   
>>       def remove(self, pkgs, with_dependencies=True):
>>           if with_dependencies:
>> -            cmd = "%s %s remove %s" % \
>> +            cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
>>                   (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
>>           else:
>>               cmd = "%s %s --force-depends remove %s" % \
>>                   (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
>>   
>>           try:
>> -            subprocess.check_output(cmd.split())
>> +            bb.note(cmd)
>> +            output = subprocess.check_output(cmd.split())
>> +            bb.note(output)
>>           except subprocess.CalledProcessError as e:
>>               bb.fatal("Unable to remove packages. Command '%s' "
>>                        "returned %d:\n%s" % (e.cmd, e.returncode, e.output))
>> @@ -1175,6 +1187,92 @@ class OpkgPM(PackageManager):
>>                       else:
>>                           status.write(line + "\n")
>>   
>> +    '''
>> +    If incremental install, we need to determine what we've got,
>> +    what we need to add, and what to remove...
>> +    The dump_install_solution will dump and save the new install
>> +    solution.
>> +    '''
>> +    def dump_install_solution(self, pkgs):
> Why not have this function in the Manifest class? We have an API in
> place: Manifest.create_final() that should probably take care of
> anything related to manifest(s) creation after all packages are installed.

Got it,

The dump_install_solution creates a manifest with the
packages that *will be* installed into the image, it is a
dummy installation.

I will move 'dump_install_solution' to Manifest class as
create_full.

>> +        bb.note('creating new install solution for incremental install')
>> +        if len(pkgs) == 0:
>> +            return
>> +
>> +        install_pkgs = list()
>> +
>> +        # Create an temp dir as opkg root for simulating the installation
>> +        temp_rootfs = self.d.expand('${T}/opkg')
>> +        temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg')
>> +        bb.utils.mkdirhier(temp_opkg_dir)
>> +
>> +        opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
>> +        opkg_args += self.d.getVar("OPKG_ARGS", True)
>> +
>> +        cmd = "%s %s update" % (self.opkg_cmd,
>> +                                opkg_args)
>> +        try:
>> +            subprocess.check_output(cmd, shell=True)
>> +        except subprocess.CalledProcessError as e:
>> +            bb.note("Unable to dump install packages. Command '%s' "
>> +                    "returned %d:\n%s" % (cmd, e.returncode, e.output))
>> +
>> +        # Simulate installation from zero
>> +        cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
>> +                                                opkg_args,
>> +                                                ' '.join(pkgs))
>> +        cmd += "| awk '/^Installing/{print $2}' "
>> +        cmd += "| sort -u -o %s" % self.solution_manifest
> Do we need to preserve this bash one-liner? Why not handle the command
> output using Python?

Agree, I will handle this in Python.

>> +        try:
>> +            subprocess.check_output(cmd, shell=True)
>> +            with open(self.solution_manifest, 'r') as manifest:
>> +                for pkg in manifest.read().split('\n'):
>> +                    install_pkgs.append(pkg)
>> +        except subprocess.CalledProcessError as e:
>> +            bb.note("Unable to dump install packages. Command '%s' "
>> +                    "returned %d:\n%s" % (cmd, e.returncode, e.output))
>> +
>> +        bb.utils.remove(temp_rootfs, True)
>> +
>> +        return install_pkgs
>> +
>> +    '''
>> +    If incremental install, we need to determine what we've got,
>> +    what we need to add, and what to remove...
>> +    The load_old_install_solution will load the previous install
>> +    solution
>> +    '''
>> +    def load_old_install_solution(self):
> Same here: why not put it in the Manifest class? As we already have the
> Manifest.parse_initial_manifest(), we can have something similar for the
> final_manifest (or install_solution, etc) parsing.

Agree, Manifest.parse_full_manifest() will be added

>> +        bb.note('load old install solution for incremental install')
>> +        installed_pkgs = list()
>> +        if not os.path.exists(self.solution_manifest):
>> +            bb.note('old install solution not exist')
>> +            return installed_pkgs
>> +
>> +        with open(self.solution_manifest, 'r') as manifest:
>> +            for pkg in manifest.read().split('\n'):
>> +                installed_pkgs.append(pkg.strip())
>> +
>> +        return installed_pkgs
>> +
>> +    def backup_packaging_data(self):
>> +        # Save the opkglib for increment rpm image generation
> s/rpm/opkg/

Sorry for the typo

>> +        if os.path.exists(self.saved_opkg_dir):
>> +            bb.utils.remove(self.saved_opkg_dir, True)
>> +        shutil.copytree(self.opkg_dir,
>> +                        self.saved_opkg_dir,
>> +                        symlinks=True)
>> +
>> +    def recovery_packaging_data(self):
> s/recovery/recover/ ?

Sorry for the typo

V2 incoming

//Hongxu

>> +        # Move the opkglib back
>> +        if os.path.exists(self.saved_opkg_dir):
>> +            if os.path.exists(self.opkg_dir):
>> +                bb.utils.remove(self.opkg_dir, True)
>> +
>> +            bb.note('Recovery packaging data')
>> +            shutil.copytree(self.saved_opkg_dir,
>> +                            self.opkg_dir,
>> +                            symlinks=True)
>> +
>>   
>>   class DpkgPM(PackageManager):
>>       def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):
>> -- 
>> 1.8.1.2
>>

Patch

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 6dc8fbd..4ea9677 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -934,12 +934,13 @@  class RpmPM(PackageManager):
 
 
 class OpkgPM(PackageManager):
-    def __init__(self, d, target_rootfs, config_file, archs):
+    def __init__(self, d, target_rootfs, config_file, archs, task_name='target'):
         super(OpkgPM, self).__init__(d)
 
         self.target_rootfs = target_rootfs
         self.config_file = config_file
         self.pkg_archs = archs
+        self.task_name = task_name
 
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
@@ -956,6 +957,13 @@  class OpkgPM(PackageManager):
 
         bb.utils.mkdirhier(self.opkg_dir)
 
+        self.solution_manifest = self.d.expand('${T}/saved/%s_solution' %
+                                               self.task_name)
+        self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
+        if not os.path.exists(self.d.expand('${T}/saved')):
+            bb.utils.mkdirhier(self.d.expand('${T}/saved'))
+
+
         if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS', True) or "") != "1":
             self._create_config()
         else:
@@ -1075,7 +1083,9 @@  class OpkgPM(PackageManager):
 
         try:
             bb.note("Installing the following packages: %s" % ' '.join(pkgs))
-            subprocess.check_output(cmd.split())
+            bb.note(cmd)
+            output = subprocess.check_output(cmd.split())
+            bb.note(output)
         except subprocess.CalledProcessError as e:
             (bb.fatal, bb.note)[attempt_only]("Unable to install packages. "
                                               "Command '%s' returned %d:\n%s" %
@@ -1083,14 +1093,16 @@  class OpkgPM(PackageManager):
 
     def remove(self, pkgs, with_dependencies=True):
         if with_dependencies:
-            cmd = "%s %s remove %s" % \
+            cmd = "%s %s --force-depends --force-remove --force-removal-of-dependent-packages remove %s" % \
                 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
         else:
             cmd = "%s %s --force-depends remove %s" % \
                 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
 
         try:
-            subprocess.check_output(cmd.split())
+            bb.note(cmd)
+            output = subprocess.check_output(cmd.split())
+            bb.note(output)
         except subprocess.CalledProcessError as e:
             bb.fatal("Unable to remove packages. Command '%s' "
                      "returned %d:\n%s" % (e.cmd, e.returncode, e.output))
@@ -1175,6 +1187,92 @@  class OpkgPM(PackageManager):
                     else:
                         status.write(line + "\n")
 
+    '''
+    If incremental install, we need to determine what we've got,
+    what we need to add, and what to remove...
+    The dump_install_solution will dump and save the new install
+    solution.
+    '''
+    def dump_install_solution(self, pkgs):
+        bb.note('creating new install solution for incremental install')
+        if len(pkgs) == 0:
+            return
+
+        install_pkgs = list()
+
+        # Create an temp dir as opkg root for simulating the installation
+        temp_rootfs = self.d.expand('${T}/opkg')
+        temp_opkg_dir = os.path.join(temp_rootfs, 'var/lib/opkg')
+        bb.utils.mkdirhier(temp_opkg_dir)
+
+        opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
+        opkg_args += self.d.getVar("OPKG_ARGS", True)
+
+        cmd = "%s %s update" % (self.opkg_cmd,
+                                opkg_args)
+        try:
+            subprocess.check_output(cmd, shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.note("Unable to dump install packages. Command '%s' "
+                    "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        # Simulate installation from zero
+        cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
+                                                opkg_args,
+                                                ' '.join(pkgs))
+        cmd += "| awk '/^Installing/{print $2}' "
+        cmd += "| sort -u -o %s" % self.solution_manifest
+        try:
+            subprocess.check_output(cmd, shell=True)
+            with open(self.solution_manifest, 'r') as manifest:
+                for pkg in manifest.read().split('\n'):
+                    install_pkgs.append(pkg)
+        except subprocess.CalledProcessError as e:
+            bb.note("Unable to dump install packages. Command '%s' "
+                    "returned %d:\n%s" % (cmd, e.returncode, e.output))
+
+        bb.utils.remove(temp_rootfs, True)
+
+        return install_pkgs
+
+    '''
+    If incremental install, we need to determine what we've got,
+    what we need to add, and what to remove...
+    The load_old_install_solution will load the previous install
+    solution
+    '''
+    def load_old_install_solution(self):
+        bb.note('load old install solution for incremental install')
+        installed_pkgs = list()
+        if not os.path.exists(self.solution_manifest):
+            bb.note('old install solution not exist')
+            return installed_pkgs
+
+        with open(self.solution_manifest, 'r') as manifest:
+            for pkg in manifest.read().split('\n'):
+                installed_pkgs.append(pkg.strip())
+
+        return installed_pkgs
+
+    def backup_packaging_data(self):
+        # Save the opkglib for increment rpm image generation
+        if os.path.exists(self.saved_opkg_dir):
+            bb.utils.remove(self.saved_opkg_dir, True)
+        shutil.copytree(self.opkg_dir,
+                        self.saved_opkg_dir,
+                        symlinks=True)
+
+    def recovery_packaging_data(self):
+        # Move the opkglib back
+        if os.path.exists(self.saved_opkg_dir):
+            if os.path.exists(self.opkg_dir):
+                bb.utils.remove(self.opkg_dir, True)
+
+            bb.note('Recovery packaging data')
+            shutil.copytree(self.saved_opkg_dir,
+                            self.opkg_dir,
+                            symlinks=True)
+
 
 class DpkgPM(PackageManager):
     def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None):