Patchwork [1/2] do_rootfs: Added PACKAGE_FEED_URIS functionality

login
register
mail settings
Submitter David Nyström
Date Feb. 27, 2014, 8:20 p.m.
Message ID <1393532438-6852-1-git-send-email-david.c.nystrom@gmail.com>
Download mbox | patch
Permalink /patch/67647/
State Accepted
Commit 7af3eb8434c3347b44a906a8d557cccf2cf3ba97
Headers show

Comments

David Nyström - Feb. 27, 2014, 8:20 p.m.
Adding a common interface to add predefined package manager
channels to prebuilt rootfs:es.

Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will
assume repo directories named (rpm,ipk,deb) as subdirectories
and statically add them to the rootfs, using the same PKG_ARCHs
as the build which produced the images.

Tested with RPM, IPK and DEB.

deb feed functionality seem broken, is anyone using this ?

Signed-off-by: David Nyström <david.c.nystrom@gmail.com>
Signed-off-by: David Nyström <david.nystrom@enea.com>
---
 meta/lib/oe/package_manager.py | 89 +++++++++++++++++++++++++++++++++++++++++-
 meta/lib/oe/rootfs.py          | 16 ++------
 2 files changed, 92 insertions(+), 13 deletions(-)
Paul Eggleton - Feb. 27, 2014, 10:34 p.m.
On Thursday 27 February 2014 21:20:37 David Nyström wrote:
> Adding a common interface to add predefined package manager
> channels to prebuilt rootfs:es.
> 
> Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will
> assume repo directories named (rpm,ipk,deb) as subdirectories
> and statically add them to the rootfs, using the same PKG_ARCHs
> as the build which produced the images.
> 
> Tested with RPM, IPK and DEB.

Looks good. The only thing that looks like it might be missing here is there 
appears to be no way to name the package repo entries individually, though I'm 
not sure what syntax we would use to allow that.

Thoughts?

Cheers,
Paul
Laurentiu Palcu - Feb. 28, 2014, 7:29 a.m.
Reviewed-by: Laurentiu Palcu <laurentiu.palcu@intel.com>

On Thu, Feb 27, 2014 at 09:20:37PM +0100, David Nyström wrote:
> Adding a common interface to add predefined package manager
> channels to prebuilt rootfs:es.
> 
> Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will
> assume repo directories named (rpm,ipk,deb) as subdirectories
> and statically add them to the rootfs, using the same PKG_ARCHs
> as the build which produced the images.
> 
> Tested with RPM, IPK and DEB.
> 
> deb feed functionality seem broken, is anyone using this ?
> 
> Signed-off-by: David Nyström <david.c.nystrom@gmail.com>
> Signed-off-by: David Nyström <david.nystrom@enea.com>
> ---
>  meta/lib/oe/package_manager.py | 89 +++++++++++++++++++++++++++++++++++++++++-
>  meta/lib/oe/rootfs.py          | 16 ++------
>  2 files changed, 92 insertions(+), 13 deletions(-)
> 
> diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
> index ff4f1de..c930572 100644
> --- a/meta/lib/oe/package_manager.py
> +++ b/meta/lib/oe/package_manager.py
> @@ -223,6 +223,7 @@ class PackageManager(object):
>          self.d = d
>          self.deploy_dir = None
>          self.deploy_lock = None
> +        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
>  
>      """
>      Update the package manager package database.
> @@ -262,6 +263,10 @@ class PackageManager(object):
>      def list_installed(self, format=None):
>          pass
>  
> +    @abstractmethod
> +    def insert_feeds_uris(self):
> +        pass
> +
>      """
>      Install complementary packages based upon the list of currently installed
>      packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
> @@ -358,6 +363,46 @@ class RpmPM(PackageManager):
>  
>          self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
>  
> +
> +    def insert_feeds_uris(self):
> +        if self.feed_uris == "":
> +            return
> +
> +        # List must be prefered to least preferred order
> +        default_platform_extra = set()
> +        platform_extra = set()
> +        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
> +        for mlib in self.ml_os_list:
> +            for arch in self.ml_prefix_list[mlib]:
> +                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
> +                if mlib == bbextendvariant:
> +                        default_platform_extra.add(plt)
> +                else:
> +                        platform_extra.add(plt)
> +
> +        platform_extra = platform_extra.union(default_platform_extra)
> +
> +        arch_list = []
> +        for canonical_arch in platform_extra:
> +            arch = canonical_arch.split('-')[0]
> +            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> +                continue
> +            arch_list.append(arch)
> +
> +        uri_iterator = 0
> +        channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list)
> +
> +        for uri in self.feed_uris.split():
> +            for arch in arch_list:
> +                bb.note('Note: adding Smart channel url%d%s (%s)' %
> +                        (uri_iterator, arch, channel_priority))
> +                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
> +                                   % (uri_iterator, arch, uri, arch))
> +                self._invoke_smart('channel --set url%d-%s priority=%d' %
> +                                   (uri_iterator, arch, channel_priority))
> +                channel_priority -= 5
> +            uri_iterator += 1
> +
>      '''
>      Create configs for rpm and smart, and multilib is supported
>      '''
> @@ -944,7 +989,6 @@ class OpkgPM(PackageManager):
>  
>          self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
>          self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
> -
>          self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
>          self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
>          self.opkg_args += self.d.getVar("OPKG_ARGS", True)
> @@ -1050,6 +1094,29 @@ class OpkgPM(PackageManager):
>                      config_file.write("src oe-%s file:%s\n" %
>                                        (arch, pkgs_dir))
>  
> +    def insert_feeds_uris(self):
> +        if self.feed_uris == "":
> +            return
> +
> +        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
> +                                  % self.target_rootfs)
> +
> +        with open(rootfs_config, "w+") as config_file:
> +            uri_iterator = 0
> +            for uri in self.feed_uris.split():
> +                config_file.write("src/gz url-%d %s/ipk\n" %
> +                                  (uri_iterator, uri))
> +
> +                for arch in self.pkg_archs.split():
> +                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> +                        continue
> +                    bb.note('Note: adding opkg channel url-%s-%d (%s)' %
> +                        (arch, uri_iterator, uri))
> +
> +                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
> +                                      (arch, uri_iterator, uri, arch))
> +                uri_iterator += 1
> +
>      def update(self):
>          self.deploy_dir_lock()
>  
> @@ -1410,6 +1477,26 @@ class DpkgPM(PackageManager):
>          if result is not None:
>              bb.fatal(result)
>  
> +    def insert_feeds_uris(self):
> +        if self.feed_uris == "":
> +            return
> +
> +        sources_conf = os.path.join("%s/etc/apt/sources.list"
> +                                    % self.target_rootfs)
> +        arch_list = []
> +        archs = self.d.getVar('PACKAGE_ARCHS', True)
> +        for arch in archs.split():
> +            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
> +                continue
> +            arch_list.append(arch)
> +
> +        with open(sources_conf, "w+") as sources_file:
> +            for uri in self.feed_uris.split():
> +                for arch in arch_list:
> +                    bb.note('Note: adding dpkg channel at (%s)' % uri)
> +                    sources_file.write("deb %s/deb/%s ./\n" %
> +                                       (uri, arch))
> +
>      def _create_configs(self, archs, base_archs):
>          base_archs = re.sub("_", "-", base_archs)
>  
> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
> index be0afa6..cb0a739 100644
> --- a/meta/lib/oe/rootfs.py
> +++ b/meta/lib/oe/rootfs.py
> @@ -41,9 +41,10 @@ class Rootfs(object):
>      def _log_check(self):
>          pass
>  
> -    @abstractmethod
>      def _insert_feed_uris(self):
> -        pass
> +        if base_contains("IMAGE_FEATURES", "package-management",
> +                         True, False, self.d):
> +            self.pm.insert_feeds_uris()
>  
>      @abstractmethod
>      def _handle_intercept_failure(self, failed_script):
> @@ -349,9 +350,6 @@ class RpmRootfs(Rootfs):
>                  if found_error == 6:
>                      bb.fatal(message)
>  
> -    def _insert_feed_uris(self):
> -        pass
> -
>      def _handle_intercept_failure(self, registered_pkgs):
>          rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
>          bb.utils.mkdirhier(rpm_postinsts_dir)
> @@ -372,6 +370,7 @@ class DpkgRootfs(Rootfs):
>                           d.getVar('PACKAGE_ARCHS', True),
>                           d.getVar('DPKG_ARCH', True))
>  
> +
>      def _create(self):
>          pkgs_to_install = self.manifest.parse_initial_manifest()
>  
> @@ -432,9 +431,6 @@ class DpkgRootfs(Rootfs):
>      def _log_check(self):
>          pass
>  
> -    def _insert_feed_uris(self):
> -        pass
> -
>  
>  class OpkgRootfs(Rootfs):
>      def __init__(self, d, manifest_dir):
> @@ -698,10 +694,6 @@ class OpkgRootfs(Rootfs):
>      def _log_check(self):
>          pass
>  
> -    def _insert_feed_uris(self):
> -        pass
> -
> -
>  def create_rootfs(d, manifest_dir=None):
>      env_bkp = os.environ.copy()
>  
> -- 
> 1.8.3.2
> 
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/openembedded-core
David Nyström - Feb. 28, 2014, 9:38 a.m.
On 2014-02-27 23:34, Paul Eggleton wrote:
> On Thursday 27 February 2014 21:20:37 David Nyström wrote:
>> Adding a common interface to add predefined package manager
>> channels to prebuilt rootfs:es.
>>
>> Adding PACKAGE_FEED_URIS = "http://myre.po/repo/, will
>> assume repo directories named (rpm,ipk,deb) as subdirectories
>> and statically add them to the rootfs, using the same PKG_ARCHs
>> as the build which produced the images.
>>
>> Tested with RPM, IPK and DEB.
>
> Looks good. The only thing that looks like it might be missing here is there
> appears to be no way to name the package repo entries individually, though I'm
> not sure what syntax we would use to allow that.
>
> Thoughts?

Well, there is the old moblin syntax for IPK_FEED_URIS:

For qemux86-64
--
IPK_FEED_URIS = " \
 
all##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/all \
 
x86_64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/x86_64 
\
 
core2-64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/core2-64 
\
 
qemux86_64##http://downloads.yoctoproject.org/releases/yocto/yocto-1.5/ipk/qemux86_64 
\
--
I don't particularly agree with above format.
When using the deploy directory as input it makes the feed_syntax 
MACHINE dependent, and difficult to keep in sync for new machines and 
upstream changes.

IMHO, bitbake should support the same input format as it generates as 
output format for the repos.
If someone want to share code for the post-processing of outputted 
bitbake repos(deb,ipk,rpm), I'll happily reconsider.

Having multiple feed formats is confusing though, we should fix this so 
we have the same format everywhere.

What could be done is to add names to the top level repos directory,
i.e. PACKAGE_FEED_URIS ="stable##http://myre.op/deploy/ \
			"experimental#http://not_trusted.to/ext_deploy/"

But I'm not sure what this will accomplish. The real channel names would 
become experimental-all, stable-all instead of current hardcoded 
url0-all, url1-all et.c.
Is this what you were suggesting ?

With smart and apt, you can prioritize channels, which the 
implementation will do, in descending order, keeping the pkg_arch 
inter-priorities intact.
However, for opkg, you prioritize archs rather than channels afaik.


> Cheers,
> Paul
>
Trevor Woerner - March 3, 2014, 8:41 p.m.
Does this resolve issue 5407?
https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
Otavio Salvador - March 4, 2014, 12:57 a.m.
On Mon, Mar 3, 2014 at 5:41 PM, Trevor Woerner
<trevor.woerner@linaro.org> wrote:
> Does this resolve issue 5407?
> https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407

Seems so :D
David Nyström - March 4, 2014, 8:45 a.m.
On 2014-03-03 21:41, Trevor Woerner wrote:
> Does this resolve issue 5407?
> https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
>

This patch satisfies a small part of 5407, when this patch has been 
reworked into a recipe and merged, there is still the issue to 
synchonize the formatting for repositories in f.ex. IPK_FEED_URIS, and a 
general cleanup of code made obsolete(poky-feed-config-ipk, distro-feeds 
et.c.)

Br,
David
Paul Eggleton - March 4, 2014, 10:29 a.m.
On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
> On 2014-03-03 21:41, Trevor Woerner wrote:
> > Does this resolve issue 5407?
> > https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
> 
> This patch satisfies a small part of 5407, when this patch has been
> reworked into a recipe and merged

I'm still confused as to why reworking this into a recipe is beneficial... ?

Cheers,
Paul
Andreas Oberritter - March 4, 2014, 11:02 a.m.
On 04.03.2014 11:29, Paul Eggleton wrote:
> On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
>> On 2014-03-03 21:41, Trevor Woerner wrote:
>>> Does this resolve issue 5407?
>>> https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
>>
>> This patch satisfies a small part of 5407, when this patch has been
>> reworked into a recipe and merged
> 
> I'm still confused as to why reworking this into a recipe is beneficial... ?

Because if the feed info is stored in a package, it can be updated
without reflashing.

We used this once to stop updates for selected machines from a shared
feed (due to memory constraints), but allowed other machines to continue
to receive updates by pointing them to a new feed location.

Regards,
Andreas
Paul Eggleton - March 4, 2014, 11:04 a.m.
On Tuesday 04 March 2014 12:02:41 Andreas Oberritter wrote:
> On 04.03.2014 11:29, Paul Eggleton wrote:
> > On Tuesday 04 March 2014 09:45:09 David Nyström wrote:
> >> On 2014-03-03 21:41, Trevor Woerner wrote:
> >>> Does this resolve issue 5407?
> >>> https://bugzilla.yoctoproject.org/show_bug.cgi?id=5407
> >> 
> >> This patch satisfies a small part of 5407, when this patch has been
> >> reworked into a recipe and merged
> > 
> > I'm still confused as to why reworking this into a recipe is beneficial...
> > ?
>
> Because if the feed info is stored in a package, it can be updated
> without reflashing.
> 
> We used this once to stop updates for selected machines from a shared
> feed (due to memory constraints), but allowed other machines to continue
> to receive updates by pointing them to a new feed location.

I see - makes sense.

Thanks,
Paul

Patch

diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index ff4f1de..c930572 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -223,6 +223,7 @@  class PackageManager(object):
         self.d = d
         self.deploy_dir = None
         self.deploy_lock = None
+        self.feed_uris = self.d.getVar('PACKAGE_FEED_URIS', True) or ""
 
     """
     Update the package manager package database.
@@ -262,6 +263,10 @@  class PackageManager(object):
     def list_installed(self, format=None):
         pass
 
+    @abstractmethod
+    def insert_feeds_uris(self):
+        pass
+
     """
     Install complementary packages based upon the list of currently installed
     packages e.g. locales, *-dev, *-dbg, etc. This will only attempt to install
@@ -358,6 +363,46 @@  class RpmPM(PackageManager):
 
         self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var)
 
+
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        # List must be prefered to least preferred order
+        default_platform_extra = set()
+        platform_extra = set()
+        bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
+        for mlib in self.ml_os_list:
+            for arch in self.ml_prefix_list[mlib]:
+                plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
+                if mlib == bbextendvariant:
+                        default_platform_extra.add(plt)
+                else:
+                        platform_extra.add(plt)
+
+        platform_extra = platform_extra.union(default_platform_extra)
+
+        arch_list = []
+        for canonical_arch in platform_extra:
+            arch = canonical_arch.split('-')[0]
+            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                continue
+            arch_list.append(arch)
+
+        uri_iterator = 0
+        channel_priority = 10 + 5 * len(self.feed_uris.split()) * len(arch_list)
+
+        for uri in self.feed_uris.split():
+            for arch in arch_list:
+                bb.note('Note: adding Smart channel url%d%s (%s)' %
+                        (uri_iterator, arch, channel_priority))
+                self._invoke_smart('channel --add url%d-%s type=rpm-md baseurl=%s/rpm/%s -y'
+                                   % (uri_iterator, arch, uri, arch))
+                self._invoke_smart('channel --set url%d-%s priority=%d' %
+                                   (uri_iterator, arch, channel_priority))
+                channel_priority -= 5
+            uri_iterator += 1
+
     '''
     Create configs for rpm and smart, and multilib is supported
     '''
@@ -944,7 +989,6 @@  class OpkgPM(PackageManager):
 
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
-
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg-cl")
         self.opkg_args = "-f %s -o %s " % (self.config_file, target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS", True)
@@ -1050,6 +1094,29 @@  class OpkgPM(PackageManager):
                     config_file.write("src oe-%s file:%s\n" %
                                       (arch, pkgs_dir))
 
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
+                                  % self.target_rootfs)
+
+        with open(rootfs_config, "w+") as config_file:
+            uri_iterator = 0
+            for uri in self.feed_uris.split():
+                config_file.write("src/gz url-%d %s/ipk\n" %
+                                  (uri_iterator, uri))
+
+                for arch in self.pkg_archs.split():
+                    if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                        continue
+                    bb.note('Note: adding opkg channel url-%s-%d (%s)' %
+                        (arch, uri_iterator, uri))
+
+                    config_file.write("src/gz uri-%s-%d %s/ipk/%s\n" %
+                                      (arch, uri_iterator, uri, arch))
+                uri_iterator += 1
+
     def update(self):
         self.deploy_dir_lock()
 
@@ -1410,6 +1477,26 @@  class DpkgPM(PackageManager):
         if result is not None:
             bb.fatal(result)
 
+    def insert_feeds_uris(self):
+        if self.feed_uris == "":
+            return
+
+        sources_conf = os.path.join("%s/etc/apt/sources.list"
+                                    % self.target_rootfs)
+        arch_list = []
+        archs = self.d.getVar('PACKAGE_ARCHS', True)
+        for arch in archs.split():
+            if not os.path.exists(os.path.join(self.deploy_dir, arch)):
+                continue
+            arch_list.append(arch)
+
+        with open(sources_conf, "w+") as sources_file:
+            for uri in self.feed_uris.split():
+                for arch in arch_list:
+                    bb.note('Note: adding dpkg channel at (%s)' % uri)
+                    sources_file.write("deb %s/deb/%s ./\n" %
+                                       (uri, arch))
+
     def _create_configs(self, archs, base_archs):
         base_archs = re.sub("_", "-", base_archs)
 
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index be0afa6..cb0a739 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -41,9 +41,10 @@  class Rootfs(object):
     def _log_check(self):
         pass
 
-    @abstractmethod
     def _insert_feed_uris(self):
-        pass
+        if base_contains("IMAGE_FEATURES", "package-management",
+                         True, False, self.d):
+            self.pm.insert_feeds_uris()
 
     @abstractmethod
     def _handle_intercept_failure(self, failed_script):
@@ -349,9 +350,6 @@  class RpmRootfs(Rootfs):
                 if found_error == 6:
                     bb.fatal(message)
 
-    def _insert_feed_uris(self):
-        pass
-
     def _handle_intercept_failure(self, registered_pkgs):
         rpm_postinsts_dir = self.image_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/')
         bb.utils.mkdirhier(rpm_postinsts_dir)
@@ -372,6 +370,7 @@  class DpkgRootfs(Rootfs):
                          d.getVar('PACKAGE_ARCHS', True),
                          d.getVar('DPKG_ARCH', True))
 
+
     def _create(self):
         pkgs_to_install = self.manifest.parse_initial_manifest()
 
@@ -432,9 +431,6 @@  class DpkgRootfs(Rootfs):
     def _log_check(self):
         pass
 
-    def _insert_feed_uris(self):
-        pass
-
 
 class OpkgRootfs(Rootfs):
     def __init__(self, d, manifest_dir):
@@ -698,10 +694,6 @@  class OpkgRootfs(Rootfs):
     def _log_check(self):
         pass
 
-    def _insert_feed_uris(self):
-        pass
-
-
 def create_rootfs(d, manifest_dir=None):
     env_bkp = os.environ.copy()