Patchwork [bitbake-devel,oe-core,RFC,1/2] fetch2: Add an extra check for sstate_mirrors

login
register
mail settings
Submitter Joe MacDonald
Date June 28, 2013, 3:05 p.m.
Message ID <1372431907-10142-2-git-send-email-joe.macdonald@windriver.com>
Download mbox | patch
Permalink /patch/52567/
State New
Headers show

Comments

Joe MacDonald - June 28, 2013, 3:05 p.m.
BB_NO_NETWORK disables any fetching, however if we're using an external
sstate cache, we may want to be able to fetch those objects even if we are
not fetching the upstream sources.

Signed-off-by: Joe MacDonald <joe.macdonald@windriver.com>
---
 bitbake/lib/bb/fetch2/__init__.py |   18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)
Richard Purdie - June 28, 2013, 4:56 p.m.
On Fri, 2013-06-28 at 11:05 -0400, Joe MacDonald wrote:
> BB_NO_NETWORK disables any fetching, however if we're using an external
> sstate cache, we may want to be able to fetch those objects even if we are
> not fetching the upstream sources.
> 
> Signed-off-by: Joe MacDonald <joe.macdonald@windriver.com>
> ---
>  bitbake/lib/bb/fetch2/__init__.py |   18 +++++++++++++-----
>  1 file changed, 13 insertions(+), 5 deletions(-)


Bitbake's fetch module knows nothing about sstate right now and I don't
see any reason to teach it about it. Why can't you just change
BB_NO_NETWORK in sstate.bbclass before we call into the fetcher?

Cheers,

Richard

> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
> index 6211cd7..edd290b3 100644
> --- a/bitbake/lib/bb/fetch2/__init__.py
> +++ b/bitbake/lib/bb/fetch2/__init__.py
> @@ -714,10 +714,13 @@ def check_network_access(d, info = "", url = None):
>      """
>      log remote network access, and error if BB_NO_NETWORK is set
>      """
> -    if d.getVar("BB_NO_NETWORK", True) == "1":
> -        raise NetworkAccess(url, info)
> +    if d.getVar("SSTATE_MIRROR_ALLOW_NETWORK", True) == "1" and d.getVar("LOCAL_SSTATE_CACHE", True) == True:
> +        logger.debug(1, "Fetcher accessed the network for sstate cache data with the command %s" % info)
>      else:
> -        logger.debug(1, "Fetcher accessed the network with the command %s" % info)
> +        if d.getVar("BB_NO_NETWORK", True) == "1":
> +            raise NetworkAccess(url, info)
> +        else:
> +            logger.debug(1, "Fetcher accessed the network with the command %s" % info)
>  
>  def build_mirroruris(origud, mirrors, ld):
>      uris = []
> @@ -1301,7 +1304,7 @@ class FetchMethod(object):
>          return "%s-%s" % (key, d.getVar("PN", True) or "")
>  
>  class Fetch(object):
> -    def __init__(self, urls, d, cache = True, localonly = False):
> +    def __init__(self, urls, d, cache = True, localonly = False, sstate_cache = False):
>          if localonly and cache:
>              raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
>  
> @@ -1310,6 +1313,8 @@ class Fetch(object):
>          self.urls = urls
>          self.d = d
>          self.ud = {}
> +        self.sstate_cache = sstate_cache
> +        d.setVar("LOCAL_SSTATE_CACHE", sstate_cache)
>  
>          fn = d.getVar('FILE', True)
>          if cache and fn and fn in urldata_cache:
> @@ -1354,7 +1359,10 @@ class Fetch(object):
>          if len(urls) == 0:
>              urls = self.urls
>  
> -        network = self.d.getVar("BB_NO_NETWORK", True)
> +        if self.sstate_cache == True and self.d.getVar("SSTATE_MIRROR_ALLOW_NETWORK", True) == True:
> +            network = True
> +        else:
> +            network = self.d.getVar("BB_NO_NETWORK", True)
>          premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY", True) == "1")
>  
>          for u in urls:
Mark Hatle - June 28, 2013, 5 p.m.
On 6/28/13 11:56 AM, Richard Purdie wrote:
> On Fri, 2013-06-28 at 11:05 -0400, Joe MacDonald wrote:
>> BB_NO_NETWORK disables any fetching, however if we're using an external
>> sstate cache, we may want to be able to fetch those objects even if we are
>> not fetching the upstream sources.
>>
>> Signed-off-by: Joe MacDonald <joe.macdonald@windriver.com>
>> ---
>>   bitbake/lib/bb/fetch2/__init__.py |   18 +++++++++++++-----
>>   1 file changed, 13 insertions(+), 5 deletions(-)
>
>
> Bitbake's fetch module knows nothing about sstate right now and I don't
> see any reason to teach it about it. Why can't you just change
> BB_NO_NETWORK in sstate.bbclass before we call into the fetcher?

We considered that, but won't that then change the value potentially for other 
things that could be running in parallel?  Or is the namespace protection good 
enough for that?

I.e. in sstate.bbclass do a d.setVar ... call the fetch .. d.setVar (restore)?

--Mark

> Cheers,
>
> Richard
>
>> diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
>> index 6211cd7..edd290b3 100644
>> --- a/bitbake/lib/bb/fetch2/__init__.py
>> +++ b/bitbake/lib/bb/fetch2/__init__.py
>> @@ -714,10 +714,13 @@ def check_network_access(d, info = "", url = None):
>>       """
>>       log remote network access, and error if BB_NO_NETWORK is set
>>       """
>> -    if d.getVar("BB_NO_NETWORK", True) == "1":
>> -        raise NetworkAccess(url, info)
>> +    if d.getVar("SSTATE_MIRROR_ALLOW_NETWORK", True) == "1" and d.getVar("LOCAL_SSTATE_CACHE", True) == True:
>> +        logger.debug(1, "Fetcher accessed the network for sstate cache data with the command %s" % info)
>>       else:
>> -        logger.debug(1, "Fetcher accessed the network with the command %s" % info)
>> +        if d.getVar("BB_NO_NETWORK", True) == "1":
>> +            raise NetworkAccess(url, info)
>> +        else:
>> +            logger.debug(1, "Fetcher accessed the network with the command %s" % info)
>>
>>   def build_mirroruris(origud, mirrors, ld):
>>       uris = []
>> @@ -1301,7 +1304,7 @@ class FetchMethod(object):
>>           return "%s-%s" % (key, d.getVar("PN", True) or "")
>>
>>   class Fetch(object):
>> -    def __init__(self, urls, d, cache = True, localonly = False):
>> +    def __init__(self, urls, d, cache = True, localonly = False, sstate_cache = False):
>>           if localonly and cache:
>>               raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
>>
>> @@ -1310,6 +1313,8 @@ class Fetch(object):
>>           self.urls = urls
>>           self.d = d
>>           self.ud = {}
>> +        self.sstate_cache = sstate_cache
>> +        d.setVar("LOCAL_SSTATE_CACHE", sstate_cache)
>>
>>           fn = d.getVar('FILE', True)
>>           if cache and fn and fn in urldata_cache:
>> @@ -1354,7 +1359,10 @@ class Fetch(object):
>>           if len(urls) == 0:
>>               urls = self.urls
>>
>> -        network = self.d.getVar("BB_NO_NETWORK", True)
>> +        if self.sstate_cache == True and self.d.getVar("SSTATE_MIRROR_ALLOW_NETWORK", True) == True:
>> +            network = True
>> +        else:
>> +            network = self.d.getVar("BB_NO_NETWORK", True)
>>           premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY", True) == "1")
>>
>>           for u in urls:
>
>
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel
>
Richard Purdie - June 28, 2013, 5:04 p.m.
On Fri, 2013-06-28 at 12:00 -0500, Mark Hatle wrote:
> On 6/28/13 11:56 AM, Richard Purdie wrote:
> > On Fri, 2013-06-28 at 11:05 -0400, Joe MacDonald wrote:
> >> BB_NO_NETWORK disables any fetching, however if we're using an external
> >> sstate cache, we may want to be able to fetch those objects even if we are
> >> not fetching the upstream sources.
> >>
> >> Signed-off-by: Joe MacDonald <joe.macdonald@windriver.com>
> >> ---
> >>   bitbake/lib/bb/fetch2/__init__.py |   18 +++++++++++++-----
> >>   1 file changed, 13 insertions(+), 5 deletions(-)
> >
> >
> > Bitbake's fetch module knows nothing about sstate right now and I don't
> > see any reason to teach it about it. Why can't you just change
> > BB_NO_NETWORK in sstate.bbclass before we call into the fetcher?
> 
> We considered that, but won't that then change the value potentially for other 
> things that could be running in parallel?  Or is the namespace protection good 
> enough for that?
> 
> I.e. in sstate.bbclass do a d.setVar ... call the fetch .. d.setVar (restore)?

If you look carefully you'll see sstate.bbclass does a
bb.data.createCopy() of the datastore and only changes that before
calling into the fetcher module. The change is only local to that copy
and the copy only lives as long as its referenced so it should all be
fine.

Cheers,

Richard
Joe MacDonald - June 28, 2013, 7:19 p.m.
[Re: [bitbake-devel] [bitbake][oe-core][RFC PATCH 1/2] fetch2: Add an extra check for sstate_mirrors] On 13.06.28 (Fri 18:04) Richard Purdie wrote:

> On Fri, 2013-06-28 at 12:00 -0500, Mark Hatle wrote:
> > On 6/28/13 11:56 AM, Richard Purdie wrote:
> > > On Fri, 2013-06-28 at 11:05 -0400, Joe MacDonald wrote:
> > >> BB_NO_NETWORK disables any fetching, however if we're using an external
> > >> sstate cache, we may want to be able to fetch those objects even if we are
> > >> not fetching the upstream sources.
> > >>
> > >> Signed-off-by: Joe MacDonald <joe.macdonald@windriver.com>
> > >> ---
> > >>   bitbake/lib/bb/fetch2/__init__.py |   18 +++++++++++++-----
> > >>   1 file changed, 13 insertions(+), 5 deletions(-)
> > >
> > >
> > > Bitbake's fetch module knows nothing about sstate right now and I don't
> > > see any reason to teach it about it. Why can't you just change
> > > BB_NO_NETWORK in sstate.bbclass before we call into the fetcher?
> > 
> > We considered that, but won't that then change the value potentially for other 
> > things that could be running in parallel?  Or is the namespace protection good 
> > enough for that?
> > 
> > I.e. in sstate.bbclass do a d.setVar ... call the fetch .. d.setVar (restore)?
> 
> If you look carefully you'll see sstate.bbclass does a
> bb.data.createCopy() of the datastore and only changes that before
> calling into the fetcher module. The change is only local to that copy
> and the copy only lives as long as its referenced so it should all be
> fine.

Okay, thanks for the feedback, guys.  I've re-worked this into a purely
sstate.bbclass change and verified it is still behaving nicely, I'll
mail that out to openembedded-core@lists.openembedded.org now.

-J.

> 
> Cheers,
> 
> Richard
> 
> 
> 
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.openembedded.org/mailman/listinfo/bitbake-devel

Patch

diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 6211cd7..edd290b3 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -714,10 +714,13 @@  def check_network_access(d, info = "", url = None):
     """
     log remote network access, and error if BB_NO_NETWORK is set
     """
-    if d.getVar("BB_NO_NETWORK", True) == "1":
-        raise NetworkAccess(url, info)
+    if d.getVar("SSTATE_MIRROR_ALLOW_NETWORK", True) == "1" and d.getVar("LOCAL_SSTATE_CACHE", True) == True:
+        logger.debug(1, "Fetcher accessed the network for sstate cache data with the command %s" % info)
     else:
-        logger.debug(1, "Fetcher accessed the network with the command %s" % info)
+        if d.getVar("BB_NO_NETWORK", True) == "1":
+            raise NetworkAccess(url, info)
+        else:
+            logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
 def build_mirroruris(origud, mirrors, ld):
     uris = []
@@ -1301,7 +1304,7 @@  class FetchMethod(object):
         return "%s-%s" % (key, d.getVar("PN", True) or "")
 
 class Fetch(object):
-    def __init__(self, urls, d, cache = True, localonly = False):
+    def __init__(self, urls, d, cache = True, localonly = False, sstate_cache = False):
         if localonly and cache:
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
@@ -1310,6 +1313,8 @@  class Fetch(object):
         self.urls = urls
         self.d = d
         self.ud = {}
+        self.sstate_cache = sstate_cache
+        d.setVar("LOCAL_SSTATE_CACHE", sstate_cache)
 
         fn = d.getVar('FILE', True)
         if cache and fn and fn in urldata_cache:
@@ -1354,7 +1359,10 @@  class Fetch(object):
         if len(urls) == 0:
             urls = self.urls
 
-        network = self.d.getVar("BB_NO_NETWORK", True)
+        if self.sstate_cache == True and self.d.getVar("SSTATE_MIRROR_ALLOW_NETWORK", True) == True:
+            network = True
+        else:
+            network = self.d.getVar("BB_NO_NETWORK", True)
         premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY", True) == "1")
 
         for u in urls: