Patchwork sstate.bbclass: Detect overwriting files in sstate controlled directories

login
register
mail settings
Submitter Richard Purdie
Date Aug. 25, 2012, 3:48 p.m.
Message ID <1345909694.14369.113.camel@ted>
Download mbox | patch
Permalink /patch/35349/
State Accepted
Commit 1f1b6f93d6b7aa8c9bd9bb5b1826997812e36932
Headers show

Comments

Richard Purdie - Aug. 25, 2012, 3:48 p.m.
We have a potential problem where two sstate packages try and touch the same
file. This adds code which will print a warning whenever this happens.

The implementation does but by maintaining a master file list and comparing
file accesses against this. There are a number of places we have duplicate
accesses which are harmless, mostly in the deploy directory so these
are whitelisted.

For now the code prints warnings, this could be strengthened in future to
become error messages. Whilst working on this code on and off over the past
few months various issues were uncovered, some serious.

[YOCTO #238]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
Saul Wold - Aug. 27, 2012, 6:54 p.m.
On 08/25/2012 08:48 AM, Richard Purdie wrote:
> We have a potential problem where two sstate packages try and touch the same
> file. This adds code which will print a warning whenever this happens.
>
> The implementation does but by maintaining a master file list and comparing
> file accesses against this. There are a number of places we have duplicate
> accesses which are harmless, mostly in the deploy directory so these
> are whitelisted.
>
> For now the code prints warnings, this could be strengthened in future to
> become error messages. Whilst working on this code on and off over the past
> few months various issues were uncovered, some serious.
>
This is causing a failure:

IOError: [Errno 2] No such file or directory: 
'/intel/poky/builds/world/tmp/sstate-control/master.list'

<snip>

ERROR: [From file: 'sstate.bbclass', lineno: 13, function: sstate_clean]
ERROR:      0024:    # Remove the entries from the master manifest
ERROR:      0025:    mastermanifest = d.getVar("SSTATE_MASTERMANIFEST", 
True)
ERROR:      0026:    lock = bb.utils.lockfile(mastermanifest + ".lock")
ERROR:      0027:    mf = open(mastermanifest + ".new", "w")
ERROR:  *** 0028:    for line in open(mastermanifest, "r"):
ERROR:      0029:        if not line or line in entries:
ERROR:      0030:            continue
ERROR:      0031:        mf.write(line)
ERROR:      0032:    mf.close()
ERROR: [From file: 'sstate.bbclass', lineno: 28, function: 
sstate_clean_manifest]
ERROR: Function failed: sstate_task_prefunc

I see a master.list.lock and master.list.new, but no master.list.


Sau!




> [YOCTO #238]
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 6762e32..02594a7 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -3,6 +3,7 @@ SSTATE_VERSION = "2"
>   SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
>   SSTATE_MANFILEBASE = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-"
>   SSTATE_MANFILEPREFIX = "${SSTATE_MANFILEBASE}${PN}"
> +SSTATE_MASTERMANIFEST = "${SSTATE_MANIFESTS}/master.list"
>
>   def generate_sstatefn(spec, hash, d):
>       if not hash:
> @@ -17,6 +18,7 @@ SSTATE_EXTRAPATH   = ""
>   SSTATE_EXTRAPATHWILDCARD = ""
>   SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
>
> +SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/ ${DEPLOY_DIR_IPK}/all/ ${DEPLOY_DIR_RPM}/all ${DEPLOY_DIR_DEB}/all/"
>
>   SSTATE_SCAN_FILES ?= "*.la *-config *_config"
>   SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f'
> @@ -125,7 +127,6 @@ def sstate_install(ss, d):
>           locks.append(bb.utils.lockfile(lock))
>
>       for state in ss['dirs']:
> -        oe.path.copytree(state[1], state[2])
>           for walkroot, dirs, files in os.walk(state[1]):
>               bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
>               for file in files:
> @@ -140,9 +141,35 @@ def sstate_install(ss, d):
>                   if not dstdir.endswith("/"):
>                       dstdir = dstdir + "/"
>                   shareddirs.append(dstdir)
> +
> +    # Check the file list for conflicts against the master manifest
> +    mastermanifest = d.getVar("SSTATE_MASTERMANIFEST", True)
> +    whitelist = d.getVar("SSTATE_DUPWHITELIST", True)
> +    lock = bb.utils.lockfile(mastermanifest + ".lock")
> +    fileslist = [line.strip() for line in open(mastermanifest)]
> +    bb.utils.unlockfile(lock)
> +    match = []
> +    for f in sharedfiles:
> +        if f in fileslist:
> +            realmatch = True
> +            for w in whitelist:
> +                if f.startswith(w):
> +                    realmatch = False
> +                    break
> +            if realmatch:
> +                match.append(f)
> +    if match:
> +        bb.warn("The recipe is trying to install files into a shared area when those files already exist. Those files are:\n   %s" % "\n   ".join(match))
> +
> +    # Write out the manifest and add to the task's manifest file
> +    lock = bb.utils.lockfile(mastermanifest + ".lock")
> +    mf = open(mastermanifest, "a")
>       f = open(manifest, "w")
>       for file in sharedfiles:
> +        mf.write(file + "\n")
>           f.write(file + "\n")
> +    bb.utils.unlockfile(lock)
> +
>       # We want to ensure that directories appear at the end of the manifest
>       # so that when we test to see if they should be deleted any contents
>       # added by the task will have been removed first.
> @@ -152,6 +179,10 @@ def sstate_install(ss, d):
>           f.write(di + "\n")
>       f.close()
>
> +    # Run the actual file install
> +    for state in ss['dirs']:
> +        oe.path.copytree(state[1], state[2])
> +
>       for postinst in (d.getVar('SSTATEPOSTINSTFUNCS', True) or '').split():
>           bb.build.exec_func(postinst, d)
>
> @@ -268,6 +299,18 @@ def sstate_clean_manifest(manifest, d):
>           except OSError:
>               pass
>
> +    # Remove the entries from the master manifest
> +    mastermanifest = d.getVar("SSTATE_MASTERMANIFEST", True)
> +    lock = bb.utils.lockfile(mastermanifest + ".lock")
> +    mf = open(mastermanifest + ".new", "w")
> +    for line in open(mastermanifest, "r"):
> +        if not line or line in entries:
> +            continue
> +        mf.write(line)
> +    mf.close()
> +    os.rename(mastermanifest + ".new", mastermanifest)
> +    bb.utils.unlockfile(lock)
> +
>       oe.path.remove(manifest)
>
>   def sstate_clean(ss, d):
>
>
>
> _______________________________________________
> Openembedded-core mailing list
> Openembedded-core@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/openembedded-core
>

Patch

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 6762e32..02594a7 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -3,6 +3,7 @@  SSTATE_VERSION = "2"
 SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control"
 SSTATE_MANFILEBASE = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-"
 SSTATE_MANFILEPREFIX = "${SSTATE_MANFILEBASE}${PN}"
+SSTATE_MASTERMANIFEST = "${SSTATE_MANIFESTS}/master.list"
 
 def generate_sstatefn(spec, hash, d):
     if not hash:
@@ -17,6 +18,7 @@  SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
 SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
 
+SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/ ${DEPLOY_DIR_IPK}/all/ ${DEPLOY_DIR_RPM}/all ${DEPLOY_DIR_DEB}/all/"
 
 SSTATE_SCAN_FILES ?= "*.la *-config *_config"
 SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f'
@@ -125,7 +127,6 @@  def sstate_install(ss, d):
         locks.append(bb.utils.lockfile(lock))
 
     for state in ss['dirs']:
-        oe.path.copytree(state[1], state[2])
         for walkroot, dirs, files in os.walk(state[1]):
             bb.debug(2, "Staging files from %s to %s" % (state[1], state[2]))
             for file in files:
@@ -140,9 +141,35 @@  def sstate_install(ss, d):
                 if not dstdir.endswith("/"):
                     dstdir = dstdir + "/"
                 shareddirs.append(dstdir)
+
+    # Check the file list for conflicts against the master manifest
+    mastermanifest = d.getVar("SSTATE_MASTERMANIFEST", True)
+    whitelist = d.getVar("SSTATE_DUPWHITELIST", True)
+    lock = bb.utils.lockfile(mastermanifest + ".lock")
+    fileslist = [line.strip() for line in open(mastermanifest)]
+    bb.utils.unlockfile(lock)
+    match = []
+    for f in sharedfiles:
+        if f in fileslist:
+            realmatch = True
+            for w in whitelist:
+                if f.startswith(w):
+                    realmatch = False
+                    break
+            if realmatch:
+                match.append(f)
+    if match:
+        bb.warn("The recipe is trying to install files into a shared area when those files already exist. Those files are:\n   %s" % "\n   ".join(match))
+
+    # Write out the manifest and add to the task's manifest file
+    lock = bb.utils.lockfile(mastermanifest + ".lock")
+    mf = open(mastermanifest, "a")
     f = open(manifest, "w")
     for file in sharedfiles:
+        mf.write(file + "\n")
         f.write(file + "\n")
+    bb.utils.unlockfile(lock)
+
     # We want to ensure that directories appear at the end of the manifest
     # so that when we test to see if they should be deleted any contents
     # added by the task will have been removed first.
@@ -152,6 +179,10 @@  def sstate_install(ss, d):
         f.write(di + "\n")
     f.close()
 
+    # Run the actual file install
+    for state in ss['dirs']:
+        oe.path.copytree(state[1], state[2])
+
     for postinst in (d.getVar('SSTATEPOSTINSTFUNCS', True) or '').split():
         bb.build.exec_func(postinst, d)
 
@@ -268,6 +299,18 @@  def sstate_clean_manifest(manifest, d):
         except OSError:
             pass
 
+    # Remove the entries from the master manifest
+    mastermanifest = d.getVar("SSTATE_MASTERMANIFEST", True)
+    lock = bb.utils.lockfile(mastermanifest + ".lock")
+    mf = open(mastermanifest + ".new", "w")
+    for line in open(mastermanifest, "r"):
+        if not line or line in entries:
+            continue
+        mf.write(line)
+    mf.close()
+    os.rename(mastermanifest + ".new", mastermanifest)
+    bb.utils.unlockfile(lock)
+
     oe.path.remove(manifest)
 
 def sstate_clean(ss, d):