Patchwork [RFC] archiver: Use subdirectory in WORKDIR when creating archives

login
register
mail settings
Submitter Martin Jansa
Date Dec. 21, 2013, 6:56 p.m.
Message ID <1387652193-25373-1-git-send-email-Martin.Jansa@gmail.com>
Download mbox | patch
Permalink /patch/63659/
State New
Headers show

Comments

Martin Jansa - Dec. 21, 2013, 6:56 p.m.
From: Martin Jansa <martin.jansa@lge.com>

* RFC: because I haven't tested all possible archiver configurations
       please check added note about diffgz archives and .org -
       was it ever working correctly? It was like this since initial
       commit.

* archiver can sometimes overwrite or remove some files which are
  needed by other tasks.

  This happens in cases where file referenced in SRC_URI aren't
  unpacked and are used by other tasks directly from WORKDIR

  def archive_sources(d, stage_name): downloads sources from
  SRC_URI again to WORKDIR in order to create source tarball from them
  (normally it's ok, because source tarball is directly unpacked from
  DL_DIR so never stays in WORKDIR).

  def archive_sources_patches(d, stage_name): then creates the tarball
  itself and moves source and patch tarballs to deploy with:
  move_tarball_deploy(d, [source_tar_name, patch_tar_name])

  But in cases where source_tar_name == "required" file we're using:
    1) do_unpack downloads "required" from DL_DIR to WORKDIR,
    2) archive_sources overwrites it with another copy,
    3) archive_sources_patches removes it,
    4) do_deploy fails because it wants to install now non-existent
       ${WORKDIR}/required

* move duplicated ARCHIVE_SSTATE_* assignments from archive-*.bbclass
  to common archiver.bbclass
* introduce separate workdir for archiver ARCHIVE_SSTATE_WORKDIR and
  move ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR,
  ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR into it
* change archiver.bbclass to dump scripts-logs and diffgs-envdata into
  right directory (respect variables defined above instead of hardcoding
  the paths in code).
* use d.getVar* API
* unify workdir/work_dir usage for ${WORKDIR} and use separate variable
  archiver_workdir when it means ARCHIVE_SSTATE_WORKDIR

Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com>
---
 meta/classes/archive-configured-source.bbclass |   4 -
 meta/classes/archive-original-source.bbclass   |   4 -
 meta/classes/archive-patched-source.bbclass    |   4 -
 meta/classes/archiver.bbclass                  | 108 ++++++++++++++-----------
 4 files changed, 61 insertions(+), 59 deletions(-)

Patch

diff --git a/meta/classes/archive-configured-source.bbclass b/meta/classes/archive-configured-source.bbclass
index 961a532..7f6d9ee 100644
--- a/meta/classes/archive-configured-source.bbclass
+++ b/meta/classes/archive-configured-source.bbclass
@@ -40,10 +40,6 @@  python () {
         d.appendVarFlag('do_build', 'depends', build_deps)
 }
 
-ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
-ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${WORKDIR}/script-logs/"
-ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${WORKDIR}/diffgz-envdata/"
-
 SSTATETASKS += "do_archive_scripts_logs"
 do_archive_scripts_logs[sstate-inputdirs] = "${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}"
 do_archive_scripts_logs[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
diff --git a/meta/classes/archive-original-source.bbclass b/meta/classes/archive-original-source.bbclass
index e271a08..d49340d 100644
--- a/meta/classes/archive-original-source.bbclass
+++ b/meta/classes/archive-original-source.bbclass
@@ -40,10 +40,6 @@  python () {
         d.appendVarFlag('do_build', 'depends', build_deps)
 }
 
-ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
-ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${WORKDIR}/script-logs/"
-ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${WORKDIR}/diffgz-envdata/"
-
 SSTATETASKS += "do_archive_scripts_logs"
 do_archive_scripts_logs[sstate-inputdirs] = "${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}"
 do_archive_scripts_logs[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
diff --git a/meta/classes/archive-patched-source.bbclass b/meta/classes/archive-patched-source.bbclass
index c5f5cca..f7dfad1 100644
--- a/meta/classes/archive-patched-source.bbclass
+++ b/meta/classes/archive-patched-source.bbclass
@@ -40,10 +40,6 @@  python () {
         d.appendVarFlag('do_build', 'depends', build_deps)
 }
 
-ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
-ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${WORKDIR}/script-logs/"
-ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${WORKDIR}/diffgz-envdata/"
-
 SSTATETASKS += "do_archive_scripts_logs"
 do_archive_scripts_logs[sstate-inputdirs] = "${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}"
 do_archive_scripts_logs[sstate-outputdirs] = "${ARCHIVE_SSTATE_OUTDIR}"
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 66efe7d..b5ed271 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -36,6 +36,11 @@  COPYLEFT_AVAILABLE_RECIPE_TYPES = 'target native nativesdk cross crosssdk cross-
 COPYLEFT_AVAILABLE_RECIPE_TYPES[type] = 'list'
 COPYLEFT_AVAILABLE_RECIPE_TYPES[doc] = 'Space separated list of available recipe types'
 
+ARCHIVE_SSTATE_OUTDIR = "${DEPLOY_DIR}/sources/"
+ARCHIVE_SSTATE_WORKDIR = "${WORKDIR}/archiver-sources/"
+ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR = "${ARCHIVE_SSTATE_WORKDIR}/script-logs/"
+ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR = "${ARCHIVE_SSTATE_WORKDIR}/diffgz-envdata/"
+
 def copyleft_recipe_type(d):
     for recipe_type in oe.data.typed_value('COPYLEFT_AVAILABLE_RECIPE_TYPES', d):
         if oe.utils.inherits(d, recipe_type):
@@ -91,13 +96,13 @@  def get_bb_inc(d):
 
     bbinc = []
     pat=re.compile('require\s*([^\s]*\.*)(.*)')
-    work_dir = d.getVar('WORKDIR', True)
+    logsdir = d.getVar('ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR', True)
     bbfile = d.getVar('FILE', True)
     bbdir = os.path.dirname(bbfile)
     target_sys = d.getVar('TARGET_SYS', True)
     pf = d.getVar('PF', True)
     licenses = get_licenses(d)
-    script_logs = os.path.join(work_dir, 'script-logs/'+ target_sys + '/' + licenses + '/' + pf + '/script-logs')
+    script_logs = os.path.join(logsdir, target_sys + '/' + licenses + '/' + pf + '/script-logs')
     bb_inc = os.path.join(script_logs, 'bb_inc')
     bb.utils.mkdirhier(bb_inc)
 
@@ -130,17 +135,18 @@  def get_bb_inc(d):
 
 def get_logs(d):
     """
-    create a directory "script-logs" in ${WORKDIR}
+    create a directory ${ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR}
     """
-    work_dir = d.getVar('WORKDIR', True)
+    logsdir = d.getVar('ARCHIVE_SSTATE_SCRIPTS_LOGS_INDIR', True)
+    workdir = d.getVar('WORKDIR', True)
     target_sys = d.getVar('TARGET_SYS', True)
     pf = d.getVar('PF', True)
     licenses = get_licenses(d)
-    script_logs = os.path.join(work_dir, 'script-logs/'+ target_sys + '/' + licenses + '/' + pf + '/script-logs')
+    script_logs = os.path.join(logsdir, target_sys + '/' + licenses + '/' + pf + '/script-logs')
 
     try:
         bb.utils.mkdirhier(os.path.join(script_logs, 'temp'))
-        oe.path.copytree(os.path.join(work_dir, 'temp'), os.path.join(script_logs, 'temp'))
+        oe.path.copytree(os.path.join(workdir, 'temp'), os.path.join(script_logs, 'temp'))
     except (IOError, AttributeError):
         pass
     return script_logs
@@ -154,9 +160,10 @@  def get_series(d):
 
     src_patches=[]
     pf = d.getVar('PF', True)
-    work_dir = d.getVar('WORKDIR', True)
+    workdir = d.getVar('WORKDIR', True)
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
     s = d.getVar('S', True)
-    dest = os.path.join(work_dir, pf + '-series')
+    dest = os.path.join(archiver_workdir, pf + '-series')
     shutil.rmtree(dest, ignore_errors=True)
     bb.utils.mkdirhier(dest)
 
@@ -165,7 +172,7 @@  def get_series(d):
     locals = (fetch.localpath(url) for url in fetch.urls)
     for local in locals:
         src_patches.append(local)
-    if not cmp(work_dir, s):
+    if not cmp(workdir, s):
         tmp_list = src_patches
     else:
         tmp_list = src_patches[1:]
@@ -187,8 +194,8 @@  def get_applying_patches(d):
     import shutil
 
     pf = d.getVar('PF', True)
-    work_dir = d.getVar('WORKDIR', True)
-    dest = os.path.join(work_dir, pf + '-patches')
+    workdir = d.getVar('WORKDIR', True)
+    dest = os.path.join(workdir, pf + '-patches')
     shutil.rmtree(dest, ignore_errors=True)
     bb.utils.mkdirhier(dest)
 
@@ -226,17 +233,20 @@  def get_source_from_downloads(d, stage_name):
         pass
     return ''
 
-def do_tarball(workdir, srcdir, tarname):
+def do_tarball(archiver_workdir, srcdir, tarname):
     """
-    tar "srcdir" under "workdir" to "tarname"
+    tar "srcdir" under "archiver_workdir" to "tarname"
     """
     import tarfile
 
     sav_dir = os.getcwd()
-    os.chdir(workdir)
+
+    if not os.path.exists(archiver_workdir):
+        bb.utils.mkdirhier(archiver_workdir)
+    os.chdir(archiver_workdir)
     if (len(os.listdir(srcdir))) != 0:
         tar = tarfile.open(tarname, "w:gz")
-        tar.add(srcdir)
+        tar.add(srcdir, arcname=os.path.basename(srcdir))
         tar.close()
     else:
         tarname = ''
@@ -250,20 +260,20 @@  def archive_sources_from_directory(d, stage_name):
     """
 
     s = d.getVar('S', True)
-    work_dir=d.getVar('WORKDIR', True)
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
+    workdir = d.getVar('WORKDIR', True)
     PF = d.getVar('PF', True)
     tarname = PF + '-' + stage_name + ".tar.gz"
 
-    if os.path.exists(s) and work_dir in s:
+    if os.path.exists(s) and workdir in s:
         try:
-            source_dir = os.path.join(work_dir, [ i for i in s.replace(work_dir, '').split('/') if i][0])
+            source_dir = os.path.join(workdir, [ i for i in s.replace(workdir, '').split('/') if i][0])
         except IndexError:
-            if not cmp(s, work_dir):
+            if not cmp(s, workdir):
                 return ''
     else:
         return ''
-    source = os.path.basename(source_dir)
-    return do_tarball(work_dir, source, tarname)
+    return do_tarball(archiver_workdir, source_dir, tarname)
 
 def archive_sources(d, stage_name):
     """
@@ -272,10 +282,13 @@  def archive_sources(d, stage_name):
     """
     import shutil
 
-    work_dir = d.getVar('WORKDIR', True)
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
+    if not os.path.exists(archiver_workdir):
+        bb.utils.mkdirhier(archiver_workdir)
+
     file = get_source_from_downloads(d, stage_name)
     if file:
-        shutil.copy(file, work_dir)
+        shutil.copy(file, archiver_workdir)
         file = os.path.basename(file)
     else:
         file = archive_sources_from_directory(d, stage_name)
@@ -288,12 +301,11 @@  def archive_patches(d, patchdir, series):
     import shutil
 
     s = d.getVar('S', True)
-    work_dir = d.getVar('WORKDIR', True)
-    patch_dir = os.path.basename(patchdir)
-    tarname = patch_dir + ".tar.gz"
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
+    tarname = os.path.basename(patchdir) + ".tar.gz"
     if series  == 'all' and os.path.exists(os.path.join(s, 'patches/series')):
         shutil.copy(os.path.join(s, 'patches/series'), patchdir)
-    tarname = do_tarball(work_dir, patch_dir, tarname)
+    tarname = do_tarball(archiver_workdir, patchdir, tarname)
     shutil.rmtree(patchdir, ignore_errors=True)
     return tarname
 
@@ -320,11 +332,10 @@  def archive_logs(d, logdir, bbinc=False):
     import shutil
 
     pf = d.getVar('PF', True)
-    work_dir = d.getVar('WORKDIR', True)
     log_dir =  os.path.basename(logdir)
     tarname = pf + '-' + log_dir + ".tar.gz"
     archive_dir = os.path.join( logdir, '..' )
-    tarname = do_tarball(archive_dir, log_dir, tarname)
+    tarname = do_tarball(archive_dir, logdir, tarname)
     if bbinc:
         shutil.rmtree(logdir, ignore_errors=True)
     return tarname
@@ -354,7 +365,7 @@  def move_tarball_deploy(d, tarball_list):
     target_sys = d.getVar('TARGET_SYS', True)
     pf = d.getVar('PF', True)
     licenses = get_licenses(d)
-    work_dir = d.getVar('WORKDIR', True)
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
     tar_sources = d.getVar('DEPLOY_DIR', True) + '/sources/' + target_sys + '/' + licenses + '/' + pf
     if not os.path.exists(tar_sources):
         bb.utils.mkdirhier(tar_sources)
@@ -362,7 +373,7 @@  def move_tarball_deploy(d, tarball_list):
         if source:
             if os.path.exists(os.path.join(tar_sources, source)):
                 os.remove(os.path.join(tar_sources, source))
-            shutil.move(os.path.join(work_dir, source), tar_sources)
+            shutil.move(os.path.join(archiver_workdir, source), tar_sources)
 
 def check_archiving_type(d):
     """check the type for archiving package('tar' or 'srpm')"""
@@ -381,8 +392,8 @@  def get_package(d):
     """
     get tarballs name from "tar-package"
     """
-    work_dir = (d.getVar('WORKDIR', True))
-    tarlist = os.path.join(work_dir, 'tar-package')
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
+    tarlist = os.path.join(archiver_workdir, 'tar-package')
     if os.path.exists(tarlist):
         f = open(tarlist, 'r')
         line = f.readline().rstrip('\n').split()
@@ -429,8 +440,8 @@  def archive_scripts_logs(d):
     """
     import shutil
 
-    work_dir = d.getVar('WORKDIR', True)
-    temp_dir = os.path.join(work_dir, 'temp')
+    workdir = d.getVar('WORKDIR', True)
+    temp_dir = os.path.join(workdir, 'temp')
     source_archive_log_with_scripts = d.getVar('SOURCE_ARCHIVE_LOG_WITH_SCRIPTS', True)
     if source_archive_log_with_scripts == 'logs_with_scripts':
         logdir = get_logs(d)
@@ -451,13 +462,13 @@  def dumpdata(d):
     kinds of variables and functions when running a task
     """
 
-    workdir = bb.data.getVar('WORKDIR', d, 1)
-    distro = bb.data.getVar('DISTRO', d, 1)
+    diffgzdir = d.getVar('ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR', True)
+    distro = d.getVar('DISTRO', True)
     s = d.getVar('S', True)
     pf = d.getVar('PF', True)
     target_sys = d.getVar('TARGET_SYS', True)
     licenses = get_licenses(d)
-    dumpdir = os.path.join(workdir, 'diffgz-envdata/'+ target_sys + '/' + licenses + '/' + pf )
+    dumpdir = os.path.join(diffgzdir, target_sys + '/' + licenses + '/' + pf )
     if not os.path.exists(dumpdir):
         bb.utils.mkdirhier(dumpdir)
 
@@ -469,27 +480,30 @@  def dumpdata(d):
     bb.data.emit_env(f, d, True)
     # emit the metadata which isn't valid shell
     for e in d.keys():
-        if bb.data.getVarFlag(e, 'python', d):
-            f.write("\npython %s () {\n%s}\n" % (e, bb.data.getVar(e, d, 1)))
+        if d.getVarFlag(e, 'python'):
+            f.write("\npython %s () {\n%s}\n" % (e, d.getVar(e, True)))
     f.close()
 
 def create_diff_gz(d):
     """
-    creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for
+    creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz for
     mapping all content in 's' including patches to  xxx.diff.gz
+    Because "s + .'org'" doesn't exist it creates diff.gz including
+    not only changes, but also original files.
     """
     import shutil
     import subprocess
 
-    work_dir = d.getVar('WORKDIR', True)
+    workdir = d.getVar('WORKDIR', True)
+    diffgzdir = d.getVar('ARCHIVE_SSTATE_DIFFGZ_ENVDATA_INDIR', True)
     exclude_from = d.getVar('ARCHIVE_EXCLUDE_FROM', True).split()
     pf = d.getVar('PF', True)
     licenses = get_licenses(d)
     target_sys = d.getVar('TARGET_SYS', True)
-    diff_dir = os.path.join(work_dir, 'diffgz-envdata/'+ target_sys + '/' + licenses + '/' + pf )
+    diff_dir = os.path.join(diffgzdir, target_sys + '/' + licenses + '/' + pf)
     diff_file = os.path.join(diff_dir, bb.data.expand("${P}-${PR}.diff.gz",d))
 
-    f = open(os.path.join(work_dir,'temp/exclude-from-file'), 'a')
+    f = open(os.path.join(workdir,'temp/exclude-from-file'), 'a')
     for i in exclude_from:
         f.write(i)
         f.write("\n")
@@ -508,7 +522,7 @@  def create_diff_gz(d):
                 subprocess.call('fakeroot cp -rf ' + i + " " + dest, shell=True)
 
     bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz")
-    cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' +  s + " | gzip -c > " + diff_file
+    cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + workdir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' +  s + " | gzip -c > " + diff_file
     d.setVar('DIFF', cmd + "\n")
     d.setVarFlag('DIFF', 'func', '1')
     bb.build.exec_func('DIFF', d)
@@ -560,8 +574,8 @@  do_kernel_checkout[postfuncs] += "do_archive_linux_yocto "
 
 # remove tarball for sources, patches and logs after creating srpm.
 python do_delete_tarlist(){
-    work_dir = d.getVar('WORKDIR', True)
-    tarlist = os.path.join(work_dir, 'tar-package')
+    archiver_workdir = d.getVar('ARCHIVE_SSTATE_WORKDIR', True)
+    tarlist = os.path.join(archiver_workdir, 'tar-package')
     if os.path.exists(tarlist):
         os.remove(tarlist)
 }