[1/3] devtool modify: Update devtool modify to copy source from work-shared if its already downloaded.

Submitted by Sai Hari Chandana Kalluri on Dec. 5, 2018, 12:37 a.m. | Patch ID: 156910

Details

Message ID 1543970227-15416-2-git-send-email-chandana.kalluri@xilinx.com
State New
Headers show

Commit Message

Sai Hari Chandana Kalluri Dec. 5, 2018, 12:37 a.m.
In the regular devtool modify flow, the kernel source is fetched by running
do_fetch task. This is an overhead in time and space.

This patch updates modify command to check if the kernel source is already
downloaded. If so, then instead of calling do_fetch, copy the source from
work-shared to devtool workspace by creating hard links to be more efficient.
Else run the usual devtool modify flow and call do_fetch task.

[YOCTO #10416]

Signed-off-by: Sai Hari Chandana Kalluri <chandana.kalluri@xilinx.com>
Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr@xilinx.com>
---
 scripts/lib/devtool/standard.py | 124 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 120 insertions(+), 4 deletions(-)

Patch hide | download patch | download mbox

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index d14b7a6..3a8222a 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -712,6 +712,53 @@  def _check_preserve(config, recipename):
                     tf.write(line)
     os.rename(newfile, origfile)
 
+# Function links a file from src location to dest location
+def copy_file(c,dest):
+    import errno
+    destdir = os.path.dirname(dest)
+    if os.path.islink(c):
+       linkto = os.readlink(c)
+       if os.path.lexists(dest):
+            if not os.path.islink(dest):
+                raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest)
+            if os.readlink(dest) == linkto:
+                return dest
+            raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest)
+       os.symlink(linkto, dest)
+    else:
+       try:
+           os.link(c, dest)
+       except OSError as err:
+           if err.errno == errno.EXDEV:
+                bb.utils.copyfile(c, dest)
+           else:
+                raise
+
+# Function creates folders in a given target location
+def copy_dirs(root,dirs,target):
+    for d in dirs:
+        destdir =  os.path.join(target,d)
+        if os.path.islink(os.path.join(root,d)):
+             linkto = os.readlink(os.path.join(root,d))
+             os.symlink(linkto,destdir) 
+        else:
+             bb.utils.mkdirhier(target+d)
+
+# Function to link src dir to dest dir
+def copy_src_to_ws(srcdir,srctree):
+    target = srctree
+    if os.path.exists(target):
+        raise DevtoolError('source already in your workspace')
+
+    bb.utils.mkdirhier(target)
+    for root,dirs,files in os.walk(srcdir):
+        #convert abspath to relpath for root
+        destdir = root.replace(srcdir,"")
+        target = srctree+destdir+"/"
+        copy_dirs(root,dirs,target)  
+        for f in files:
+           copy_file(os.path.join(root,f),os.path.join(target,f))
+
 def modify(args, config, basepath, workspace):
     """Entry point for the devtool 'modify' subcommand"""
     import bb
@@ -758,6 +805,73 @@  def modify(args, config, basepath, workspace):
         initial_rev = None
         commits = []
         check_commits = False
+
+        if bb.data.inherits_class('kernel-yocto', rd):
+               srcdir = rd.getVar('STAGING_KERNEL_DIR')
+               if os.path.exists(srcdir) and os.listdir(srcdir):
+                  copy_src_to_ws(srcdir,srctree)
+
+                  workdir = rd.getVar('WORKDIR')
+                  srcsubdir = rd.getVar('S')
+                  localfilesdir = os.path.join(srctree,'oe-local-files') 
+                  # Move local source files into separate subdir
+                  recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
+                  local_files = oe.recipeutils.get_recipe_local_files(rd)
+
+                  for key in local_files.copy():
+                        if key.endswith('scc'):
+                              sccfile = open(local_files[key], 'r')
+                              for l in sccfile:
+                                  line = l.split()
+                                  if line and line[0] in ('kconf', 'patch'):
+                                        local_files[line[-1]] = os.path.join(os.path.dirname(local_files[key]), line[-1])
+                                        shutil.copy2(os.path.join(os.path.dirname(local_files[key]), line[-1]), workdir)
+                              sccfile.close()
+
+                  # Ignore local files with subdir={BP}
+                  srcabspath = os.path.abspath(srcsubdir)
+                  local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and  (srcabspath == workdir or not  os.path.join(workdir, fname).startswith(srcabspath + os.sep))]
+                  if local_files:
+                       for fname in local_files:
+                              copy_src_to_ws(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname))
+                       with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f:
+                              f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n')
+
+                  if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
+                  # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
+                  # (otherwise the recipe won't build as expected)
+                        local_files_dir = os.path.join(srctree, 'oe-local-files')
+                        addfiles = []
+                        for root, _, files in os.walk(local_files_dir):
+                            relpth = os.path.relpath(root, local_files_dir)
+                            if relpth != '.':
+                                  bb.utils.mkdirhier(os.path.join(srctree, relpth))
+                            for fn in files:
+                                if fn == '.gitignore':
+                                    continue
+                                destpth = os.path.join(srctree, relpth, fn)
+                                if os.path.exists(destpth):
+                                    os.unlink(destpth)
+                                os.symlink('oe-local-files/%s' % fn, destpth)
+                                addfiles.append(os.path.join(relpth, fn))
+                        if addfiles:
+                           bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree)
+                        useroptions = []
+                        oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
+                        bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
+
+                  task = 'do_configure'
+                  res = tinfoil.build_targets(pn, task, handle_events=True)
+
+                  # Copy .config to workspace 
+                  kconfpath=rd.getVar('B')
+                  logger.info('Copying kernel config to workspace')
+                  shutil.copy2(os.path.join(kconfpath, '.config'),srctree)
+
+                  # Set this to true, we still need to get initial_rev
+                  # by parsing the git repo
+                  args.no_extract = True
+
         if not args.no_extract:
             initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
             if not initial_rev:
@@ -843,10 +957,12 @@  def modify(args, config, basepath, workspace):
                 f.write('\ndo_patch() {\n'
                         '    :\n'
                         '}\n')
-                f.write('\ndo_configure_append() {\n'
-                        '    cp ${B}/.config ${S}/.config.baseline\n'
-                        '    ln -sfT ${B}/.config ${S}/.config.new\n'
-                        '}\n')
+
+            if rd.getVarFlag('do_menuconfig','task'):
+                    f.write('\ndo_configure_append() {\n'
+                    '    cp ${B}/.config ${S}/.config.baseline\n'
+                    '    ln -sfT ${B}/.config ${S}/.config.new\n'
+                    '}\n')
             if initial_rev:
                 f.write('\n# initial_rev: %s\n' % initial_rev)
                 for commit in commits:

Comments

Anuj Mittal Dec. 6, 2018, 2:23 p.m.
I gave this a try and it looks like if I change my kernel version after
one try of devtool modify, this change copies the old kernel version
tree (because something is already present in work-shared). 

And, work-shared is getting re-populated with the requested second
kernel vesion when running do_configure in the same code block later
on.

Thanks,
Anuj

On Tue, 2018-12-04 at 16:37 -0800, Sai Hari Chandana Kalluri wrote:
> In the regular devtool modify flow, the kernel source is fetched by
> running
> do_fetch task. This is an overhead in time and space.
> 
> This patch updates modify command to check if the kernel source is
> already
> downloaded. If so, then instead of calling do_fetch, copy the source
> from
> work-shared to devtool workspace by creating hard links to be more
> efficient.
> Else run the usual devtool modify flow and call do_fetch task.
> 
> [YOCTO #10416]
> 
> Signed-off-by: Sai Hari Chandana Kalluri <chandana.kalluri@xilinx.com
> >
> Signed-off-by: Alejandro Enedino Hernandez Samaniego <
> alejandr@xilinx.com>
> ---
>  scripts/lib/devtool/standard.py | 124
> ++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 120 insertions(+), 4 deletions(-)
> 
> diff --git a/scripts/lib/devtool/standard.py
> b/scripts/lib/devtool/standard.py
> index d14b7a6..3a8222a 100644
> --- a/scripts/lib/devtool/standard.py
> +++ b/scripts/lib/devtool/standard.py
> @@ -712,6 +712,53 @@ def _check_preserve(config, recipename):
>                      tf.write(line)
>      os.rename(newfile, origfile)
>  
> +# Function links a file from src location to dest location
> +def copy_file(c,dest):
> +    import errno
> +    destdir = os.path.dirname(dest)
> +    if os.path.islink(c):
> +       linkto = os.readlink(c)
> +       if os.path.lexists(dest):
> +            if not os.path.islink(dest):
> +                raise OSError(errno.EEXIST, "Link %s already exists
> as a file" % dest, dest)
> +            if os.readlink(dest) == linkto:
> +                return dest
> +            raise OSError(errno.EEXIST, "Link %s already exists to a
> different location? (%s vs %s)" % (dest, os.readlink(dest), linkto),
> dest)
> +       os.symlink(linkto, dest)
> +    else:
> +       try:
> +           os.link(c, dest)
> +       except OSError as err:
> +           if err.errno == errno.EXDEV:
> +                bb.utils.copyfile(c, dest)
> +           else:
> +                raise
> +
> +# Function creates folders in a given target location
> +def copy_dirs(root,dirs,target):
> +    for d in dirs:
> +        destdir =  os.path.join(target,d)
> +        if os.path.islink(os.path.join(root,d)):
> +             linkto = os.readlink(os.path.join(root,d))
> +             os.symlink(linkto,destdir) 
> +        else:
> +             bb.utils.mkdirhier(target+d)
> +
> +# Function to link src dir to dest dir
> +def copy_src_to_ws(srcdir,srctree):
> +    target = srctree
> +    if os.path.exists(target):
> +        raise DevtoolError('source already in your workspace')
> +
> +    bb.utils.mkdirhier(target)
> +    for root,dirs,files in os.walk(srcdir):
> +        #convert abspath to relpath for root
> +        destdir = root.replace(srcdir,"")
> +        target = srctree+destdir+"/"
> +        copy_dirs(root,dirs,target)  
> +        for f in files:
> +           copy_file(os.path.join(root,f),os.path.join(target,f))
> +
>  def modify(args, config, basepath, workspace):
>      """Entry point for the devtool 'modify' subcommand"""
>      import bb
> @@ -758,6 +805,73 @@ def modify(args, config, basepath, workspace):
>          initial_rev = None
>          commits = []
>          check_commits = False
> +
> +        if bb.data.inherits_class('kernel-yocto', rd):
> +               srcdir = rd.getVar('STAGING_KERNEL_DIR')
> +               if os.path.exists(srcdir) and os.listdir(srcdir):
> +                  copy_src_to_ws(srcdir,srctree)
> +
> +                  workdir = rd.getVar('WORKDIR')
> +                  srcsubdir = rd.getVar('S')
> +                  localfilesdir = os.path.join(srctree,'oe-local-
> files') 
> +                  # Move local source files into separate subdir
> +                  recipe_patches = [os.path.basename(patch) for
> patch in oe.recipeutils.get_recipe_patches(rd)]
> +                  local_files =
> oe.recipeutils.get_recipe_local_files(rd)
> +
> +                  for key in local_files.copy():
> +                        if key.endswith('scc'):
> +                              sccfile = open(local_files[key], 'r')
> +                              for l in sccfile:
> +                                  line = l.split()
> +                                  if line and line[0] in ('kconf',
> 'patch'):
> +                                        local_files[line[-1]] =
> os.path.join(os.path.dirname(local_files[key]), line[-1])
> +                                        shutil.copy2(os.path.join(os
> .path.dirname(local_files[key]), line[-1]), workdir)
> +                              sccfile.close()
> +
> +                  # Ignore local files with subdir={BP}
> +                  srcabspath = os.path.abspath(srcsubdir)
> +                  local_files = [fname for fname in local_files if
> os.path.exists(os.path.join(workdir, fname)) and  (srcabspath ==
> workdir or not  os.path.join(workdir, fname).startswith(srcabspath +
> os.sep))]
> +                  if local_files:
> +                       for fname in local_files:
> +                              copy_src_to_ws(os.path.join(workdir,
> fname), os.path.join(srctree, 'oe-local-files', fname))
> +                       with open(os.path.join(srctree, 'oe-local-
> files', '.gitignore'), 'w') as f:
> +                              f.write('# Ignore local files, by
> default. Remove this file ''if you want to commit the directory to
> Git\n*\n')
> +
> +                  if os.path.abspath(rd.getVar('S')) ==
> os.path.abspath(rd.getVar('WORKDIR')):
> +                  # If recipe extracts to ${WORKDIR}, symlink the
> files into the srctree
> +                  # (otherwise the recipe won't build as expected)
> +                        local_files_dir = os.path.join(srctree, 'oe-
> local-files')
> +                        addfiles = []
> +                        for root, _, files in
> os.walk(local_files_dir):
> +                            relpth = os.path.relpath(root,
> local_files_dir)
> +                            if relpth != '.':
> +                                  bb.utils.mkdirhier(os.path.join(sr
> ctree, relpth))
> +                            for fn in files:
> +                                if fn == '.gitignore':
> +                                    continue
> +                                destpth = os.path.join(srctree,
> relpth, fn)
> +                                if os.path.exists(destpth):
> +                                    os.unlink(destpth)
> +                                os.symlink('oe-local-files/%s' % fn,
> destpth)
> +                                addfiles.append(os.path.join(relpth,
> fn))
> +                        if addfiles:
> +                           bb.process.run('git add %s' % '
> '.join(addfiles), cwd=srctree)
> +                        useroptions = []
> +                        oe.patch.GitApplyTree.gitCommandUserOptions(
> useroptions, d=d)
> +                        bb.process.run('git %s commit -a -m
> "Committing local file symlinks\n\n%s"' % (' '.join(useroptions),
> oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
> +
> +                  task = 'do_configure'
> +                  res = tinfoil.build_targets(pn, task,
> handle_events=True)
> +
> +                  # Copy .config to workspace 
> +                  kconfpath=rd.getVar('B')
> +                  logger.info('Copying kernel config to workspace')
> +                  shutil.copy2(os.path.join(kconfpath,
> '.config'),srctree)
> +
> +                  # Set this to true, we still need to get
> initial_rev
> +                  # by parsing the git repo
> +                  args.no_extract = True
> +
>          if not args.no_extract:
>              initial_rev, _ = _extract_source(srctree,
> args.keep_temp, args.branch, False, config, basepath, workspace,
> args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
>              if not initial_rev:
> @@ -843,10 +957,12 @@ def modify(args, config, basepath, workspace):
>                  f.write('\ndo_patch() {\n'
>                          '    :\n'
>                          '}\n')
> -                f.write('\ndo_configure_append() {\n'
> -                        '    cp ${B}/.config
> ${S}/.config.baseline\n'
> -                        '    ln -sfT ${B}/.config
> ${S}/.config.new\n'
> -                        '}\n')
> +
> +            if rd.getVarFlag('do_menuconfig','task'):
> +                    f.write('\ndo_configure_append() {\n'
> +                    '    cp ${B}/.config ${S}/.config.baseline\n'
> +                    '    ln -sfT ${B}/.config ${S}/.config.new\n'
> +                    '}\n')
>              if initial_rev:
>                  f.write('\n# initial_rev: %s\n' % initial_rev)
>                  for commit in commits:
> -- 
> 2.7.4
>