[bitbake-devel,7/9] fetch2: provide original url in addition to the mirrored url to FetchData.__init__ and FetchMethod.urldata_init

Submitted by Urs Fässler on July 23, 2018, 3:42 p.m. | Patch ID: 152889

Details

Message ID 20180723154259.9076-8-urs.fassler@bbv.ch
State New
Headers show

Commit Message

Urs Fässler July 23, 2018, 3:42 p.m.
When a mirror rewrite rule is used, it can happen that the downloaded
file is named differently depending on the used mirror. In preparation
to a consistent naming, we provide the original url to the fetcher.

Signed-off-by: Urs Fässler <urs.fassler@bbv.ch>
Signed-off-by: Pascal Bach <pascal.bach@siemens.com>
---
 lib/bb/fetch2/__init__.py  | 12 +++++++-----
 lib/bb/fetch2/bzr.py       |  2 +-
 lib/bb/fetch2/clearcase.py |  2 +-
 lib/bb/fetch2/cvs.py       |  2 +-
 lib/bb/fetch2/git.py       |  2 +-
 lib/bb/fetch2/gitannex.py  |  4 ++--
 lib/bb/fetch2/hg.py        |  2 +-
 lib/bb/fetch2/local.py     |  2 +-
 lib/bb/fetch2/npm.py       |  2 +-
 lib/bb/fetch2/osc.py       |  2 +-
 lib/bb/fetch2/perforce.py  |  2 +-
 lib/bb/fetch2/repo.py      |  2 +-
 lib/bb/fetch2/s3.py        |  2 +-
 lib/bb/fetch2/sftp.py      |  2 +-
 lib/bb/fetch2/ssh.py       |  2 +-
 lib/bb/fetch2/svn.py       |  2 +-
 lib/bb/fetch2/wget.py      |  2 +-
 17 files changed, 24 insertions(+), 22 deletions(-)

Patch hide | download patch | download mbox

diff --git a/lib/bb/fetch2/__init__.py b/lib/bb/fetch2/__init__.py
index c8653d62..61009e43 100644
--- a/lib/bb/fetch2/__init__.py
+++ b/lib/bb/fetch2/__init__.py
@@ -932,7 +932,7 @@  def build_mirroruris(origud, mirrors, ld):
                 localmirrors.remove(line)
 
                 try:
-                    newud = FetchData(newuri, ld)
+                    newud = FetchData(newuri, ld, origud.url)
                     newud.setup_localpath(ld)
                 except bb.fetch2.BBFetchException as e:
                     logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1202,7 +1202,7 @@  class FetchData(object):
     """
     A class which represents the fetcher state for a given URI.
     """
-    def __init__(self, url, d, localonly = False):
+    def __init__(self, url, d, original_url = None, localonly = False):
         # localpath is the location of a downloaded result. If not set, the file is local.
         self.donestamp = None
         self.needdonestamp = True
@@ -1213,6 +1213,8 @@  class FetchData(object):
         self.basename = None
         self.basepath = None
         (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
+        if not original_url:
+            original_url = url
         self.date = self.getSRCDate(d)
         self.url = url
         if not self.user and "user" in self.parm:
@@ -1259,7 +1261,7 @@  class FetchData(object):
             logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN'))
             self.parm["protocol"] = self.parm.get("proto", None)
 
-        self.method.urldata_init(self, d)
+        self.method.urldata_init(self, d, original_url)
 
         if "localpath" in self.parm:
             # if user sets localpath for file, use it instead.
@@ -1348,7 +1350,7 @@  class FetchMethod(object):
 
         return True
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init method specific variable within url data
         """
@@ -1594,7 +1596,7 @@  class Fetch(object):
         for url in urls:
             if url not in self.ud:
                 try:
-                    self.ud[url] = FetchData(url, d, localonly)
+                    self.ud[url] = FetchData(url, d, localonly = localonly)
                 except NonLocalMethod:
                     if localonly:
                         self.ud[url] = None
diff --git a/lib/bb/fetch2/bzr.py b/lib/bb/fetch2/bzr.py
index 658502f9..4be73320 100644
--- a/lib/bb/fetch2/bzr.py
+++ b/lib/bb/fetch2/bzr.py
@@ -36,7 +36,7 @@  class Bzr(FetchMethod):
     def supports(self, ud, d):
         return ud.type in ['bzr']
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init bzr specific variable within url data
         """
diff --git a/lib/bb/fetch2/clearcase.py b/lib/bb/fetch2/clearcase.py
index 3a6573d0..ef1db138 100644
--- a/lib/bb/fetch2/clearcase.py
+++ b/lib/bb/fetch2/clearcase.py
@@ -84,7 +84,7 @@  class ClearCase(FetchMethod):
     def debug(self, msg):
         logger.debug(1, "ClearCase: %s", msg)
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init ClearCase specific variable within url data
         """
diff --git a/lib/bb/fetch2/cvs.py b/lib/bb/fetch2/cvs.py
index 0e0a3196..7c5296fb 100644
--- a/lib/bb/fetch2/cvs.py
+++ b/lib/bb/fetch2/cvs.py
@@ -42,7 +42,7 @@  class Cvs(FetchMethod):
         """
         return ud.type in ['cvs']
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         if not "module" in ud.parm:
             raise MissingParameterError("module", ud.url)
         ud.module = ud.parm["module"]
diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 13b9b9d8..7f7951f7 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -141,7 +141,7 @@  class Git(FetchMethod):
     def supports_checksum(self, urldata):
         return False
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init git specific variable within url data
         so that the git method like latest_revision() can work
diff --git a/lib/bb/fetch2/gitannex.py b/lib/bb/fetch2/gitannex.py
index a9b69caa..187e6b2e 100644
--- a/lib/bb/fetch2/gitannex.py
+++ b/lib/bb/fetch2/gitannex.py
@@ -33,8 +33,8 @@  class GitANNEX(Git):
         """
         return ud.type in ['gitannex']
 
-    def urldata_init(self, ud, d):
-        super(GitANNEX, self).urldata_init(ud, d)
+    def urldata_init(self, ud, d, original_url):
+        super(GitANNEX, self).urldata_init(ud, d, original_url)
         if ud.shallow:
             ud.shallow_extra_refs += ['refs/heads/git-annex', 'refs/heads/synced/*']
 
diff --git a/lib/bb/fetch2/hg.py b/lib/bb/fetch2/hg.py
index 936d0431..5fe08e80 100644
--- a/lib/bb/fetch2/hg.py
+++ b/lib/bb/fetch2/hg.py
@@ -50,7 +50,7 @@  class Hg(FetchMethod):
         """ 
         return False
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init hg specific variable within url data
         """
diff --git a/lib/bb/fetch2/local.py b/lib/bb/fetch2/local.py
index a114ac12..9e1b918c 100644
--- a/lib/bb/fetch2/local.py
+++ b/lib/bb/fetch2/local.py
@@ -39,7 +39,7 @@  class Local(FetchMethod):
         """
         return urldata.type in ['file']
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         # We don't set localfile as for this fetcher the file is already local!
         ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0])
         ud.basename = os.path.basename(ud.decodedurl)
diff --git a/lib/bb/fetch2/npm.py b/lib/bb/fetch2/npm.py
index 408dfc3d..cab792d4 100644
--- a/lib/bb/fetch2/npm.py
+++ b/lib/bb/fetch2/npm.py
@@ -60,7 +60,7 @@  class Npm(FetchMethod):
         bb.utils.remove(ud.pkgdatadir, True)
         bb.utils.remove(ud.fullmirror, False)
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init NPM specific variable within url data
         """
diff --git a/lib/bb/fetch2/osc.py b/lib/bb/fetch2/osc.py
index 6c60456b..18dd6047 100644
--- a/lib/bb/fetch2/osc.py
+++ b/lib/bb/fetch2/osc.py
@@ -25,7 +25,7 @@  class Osc(FetchMethod):
         """
         return ud.type in ['osc']
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         if not "module" in ud.parm:
             raise MissingParameterError('module', ud.url)
 
diff --git a/lib/bb/fetch2/perforce.py b/lib/bb/fetch2/perforce.py
index 903a8e61..3a15754a 100644
--- a/lib/bb/fetch2/perforce.py
+++ b/lib/bb/fetch2/perforce.py
@@ -37,7 +37,7 @@  class Perforce(FetchMethod):
         """ Check to see if a given url can be fetched with perforce. """
         return ud.type in ['p4']
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         Initialize perforce specific variables within url data.  If P4CONFIG is
         provided by the env, use it.  If P4PORT is specified by the recipe, use
diff --git a/lib/bb/fetch2/repo.py b/lib/bb/fetch2/repo.py
index 8c7e8185..c56df7e1 100644
--- a/lib/bb/fetch2/repo.py
+++ b/lib/bb/fetch2/repo.py
@@ -37,7 +37,7 @@  class Repo(FetchMethod):
         """
         return ud.type in ["repo"]
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         We don"t care about the git rev of the manifests repository, but
         we do care about the manifest to use.  The default is "default".
diff --git a/lib/bb/fetch2/s3.py b/lib/bb/fetch2/s3.py
index 16292886..e00f7072 100644
--- a/lib/bb/fetch2/s3.py
+++ b/lib/bb/fetch2/s3.py
@@ -47,7 +47,7 @@  class S3(FetchMethod):
     def recommends_checksum(self, urldata):
         return True
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         if 'downloadfilename' in ud.parm:
             ud.basename = ud.parm['downloadfilename']
         else:
diff --git a/lib/bb/fetch2/sftp.py b/lib/bb/fetch2/sftp.py
index 81884a6a..f68cbca1 100644
--- a/lib/bb/fetch2/sftp.py
+++ b/lib/bb/fetch2/sftp.py
@@ -78,7 +78,7 @@  class SFTP(FetchMethod):
     def recommends_checksum(self, urldata):
         return True
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         if 'protocol' in ud.parm and ud.parm['protocol'] == 'git':
             raise bb.fetch2.ParameterError(
                 "Invalid protocol - if you wish to fetch from a " +
diff --git a/lib/bb/fetch2/ssh.py b/lib/bb/fetch2/ssh.py
index 6047ee41..4f9e0853 100644
--- a/lib/bb/fetch2/ssh.py
+++ b/lib/bb/fetch2/ssh.py
@@ -77,7 +77,7 @@  class SSH(FetchMethod):
     def supports_checksum(self, urldata):
         return False
 
-    def urldata_init(self, urldata, d):
+    def urldata_init(self, urldata, d, original_url):
         if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git':
             raise bb.fetch2.ParameterError(
                 "Invalid protocol - if you wish to fetch from a git " +
diff --git a/lib/bb/fetch2/svn.py b/lib/bb/fetch2/svn.py
index ed70bcf8..be0e9eeb 100644
--- a/lib/bb/fetch2/svn.py
+++ b/lib/bb/fetch2/svn.py
@@ -42,7 +42,7 @@  class Svn(FetchMethod):
         """
         return ud.type in ['svn']
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         """
         init svn specific variable within url data
         """
diff --git a/lib/bb/fetch2/wget.py b/lib/bb/fetch2/wget.py
index 8f505b6d..bd2a0c6b 100644
--- a/lib/bb/fetch2/wget.py
+++ b/lib/bb/fetch2/wget.py
@@ -74,7 +74,7 @@  class Wget(FetchMethod):
     def recommends_checksum(self, urldata):
         return True
 
-    def urldata_init(self, ud, d):
+    def urldata_init(self, ud, d, original_url):
         if 'protocol' in ud.parm:
             if ud.parm['protocol'] == 'git':
                 raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url)