From patchwork Wed Jul 13 01:41:23 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Eggleton X-Patchwork-Id: 10118 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4CDB0CCA480 for ; Wed, 13 Jul 2022 01:41:33 +0000 (UTC) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mx.groups.io with SMTP id smtpd.web10.17262.1657676490637257858 for ; Tue, 12 Jul 2022 18:41:30 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linux.microsoft.com header.s=default header.b=CAyTf/9b; spf=pass (domain: linux.microsoft.com, ip: 13.77.154.182, mailfrom: pauleg@linux.microsoft.com) Received: by linux.microsoft.com (Postfix, from userid 1054) id 2A27C204DE8B; Tue, 12 Jul 2022 18:41:30 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 2A27C204DE8B DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1657676490; bh=Dw/w1fAABMixqkfLnlUucp5Xppakthvovwx4I61Ms50=; h=From:To:Subject:Date:In-Reply-To:References:From; b=CAyTf/9bwId+qffMdK0vtsCUQt999zKgV7ec3yn7QXvFuyRHBvkit/w6ivsnpZpXw 2ok/ewFdoa9Sk5MGtsDWrEJ2H2qDP8QjtMp0TqGHtdkfN7UvHZWZI+Ea4Fr7iOK1cS qDvj1nJ4zvmuVwB57SmVvrzWvKRZWkbe7/g9WWWI= From: Paul Eggleton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 1/3] patch: handle if S points to a subdirectory of a git repo Date: Tue, 12 Jul 2022 18:41:23 -0700 Message-Id: X-Mailer: git-send-email 1.8.3.1 In-Reply-To: References: List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 13 Jul 2022 01:41:33 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/167949 From: Paul Eggleton If PATCHTOOL = "git", SRC_URI fetches from a git repo and S points to a subdirectory of the checked out sources, then we were erroneously initialising the subdirectory as its own git repo. Check if the returned top-level repo directory is a subdirectory of WORKDIR and do not run initialise the source directory if that is the case. (This was a regression introduced with OE-Core revision 6184b56a7a0fc6f5d19fdfb81e7453667f7da940, however we didn't have a test that verified the behaviour.) Signed-off-by: Paul Eggleton --- meta/lib/oe/patch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py index 95b915a..4ec9cae 100644 --- a/meta/lib/oe/patch.py +++ b/meta/lib/oe/patch.py @@ -299,10 +299,10 @@ class GitApplyTree(PatchTree): PatchTree.__init__(self, dir, d) self.commituser = d.getVar('PATCH_GIT_USER_NAME') self.commitemail = d.getVar('PATCH_GIT_USER_EMAIL') - if not self._isInitialized(): + if not self._isInitialized(d): self._initRepo() - def _isInitialized(self): + def _isInitialized(self, d): cmd = "git rev-parse --show-toplevel" try: output = runcmd(cmd.split(), self.dir).strip() @@ -310,8 +310,8 @@ class GitApplyTree(PatchTree): ## runcmd returned non-zero which most likely means 128 ## Not a git directory return False - ## Make sure repo is in builddir to not break top-level git repos - return os.path.samefile(output, self.dir) + ## Make sure repo is in builddir to not break top-level git repos, or under workdir + return os.path.samefile(output, self.dir) or oe.path.is_path_parent(d.getVar('WORKDIR'), output) def _initRepo(self): runcmd("git init".split(), self.dir) From patchwork Wed Jul 13 01:41:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Eggleton X-Patchwork-Id: 10116 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 40126C433EF for ; Wed, 13 Jul 2022 01:41:33 +0000 (UTC) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mx.groups.io with SMTP id smtpd.web10.17263.1657676490654692496 for ; Tue, 12 Jul 2022 18:41:30 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linux.microsoft.com header.s=default header.b=Ty+p3kxj; spf=pass (domain: linux.microsoft.com, ip: 13.77.154.182, mailfrom: pauleg@linux.microsoft.com) Received: by linux.microsoft.com (Postfix, from userid 1054) id 35FBC20B4774; Tue, 12 Jul 2022 18:41:30 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 35FBC20B4774 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1657676490; bh=UbJIZYNfaaceP45HKhsSFjBwWw0NhSATtFo79Hw+gwg=; h=From:To:Subject:Date:In-Reply-To:References:From; b=Ty+p3kxjlVE80PRbna2R0sAjGwWYg+w/BycrzPoCGD6pLDDavwig3VCI2QiHRThSF VSDyWmz/hhMqOmfhzVl/x2mtyNxMVKsJ1nMqmWSkShn5eHpsm2V9cld2ZbY8V7lS4K uiG/K0qLCg7anQAF72Xio5/Jj/xCO9eSYzSIr8Fo= From: Paul Eggleton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 2/3] devtool: finish: handle patching when S points to subdir of a git repo Date: Tue, 12 Jul 2022 18:41:24 -0700 Message-Id: <5e1709a1dc1702470e751e4f4b307fcc4865d476.1657676159.git.paul.eggleton@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: References: List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 13 Jul 2022 01:41:33 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/167948 From: Paul Eggleton If devtool finish needs to create a patch and have it applied to the sources for a recipe where S points to a subdirectory of the sources, then the patch needs to be applied at the root of the repo i.e. we need to add a patchdir= parameter to the SRC_URI entry. Signed-off-by: Paul Eggleton --- meta/lib/oe/recipeutils.py | 9 +++++++-- scripts/lib/devtool/standard.py | 25 +++++++++++++++++++------ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 872ff97..b04992c 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -666,7 +666,7 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False): return (appendpath, pathok) -def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None): +def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None, params=None): """ Writes a bbappend file for a recipe Parameters: @@ -696,6 +696,9 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, redirect_output: If specified, redirects writing the output file to the specified directory (for dry-run purposes) + params: + Parameters to use when adding entries to SRC_URI. If specified, + should be a list of dicts with the same length as srcfiles. """ if not removevalues: @@ -762,12 +765,14 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, copyfiles = {} if srcfiles: instfunclines = [] - for newfile, origsrcfile in srcfiles.items(): + for i, (newfile, origsrcfile) in enumerate(srcfiles.items()): srcfile = origsrcfile srcurientry = None if not srcfile: srcfile = os.path.basename(newfile) srcurientry = 'file://%s' % srcfile + if params and params[i]: + srcurientry = '%s;%s' % (srcurientry, ';'.join('%s=%s' % (k,v) for k,v in params[i].items())) # Double-check it's not there already # FIXME do we care if the entry is added by another bbappend that might go away? if not srcurientry in rd.getVar('SRC_URI').split(): diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index e53569c..c98bfe8 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -1606,6 +1606,19 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil if not os.path.exists(append): raise DevtoolError('unable to find workspace bbappend for recipe %s' % recipename) + srctreebase = workspace[recipename]['srctreebase'] + relpatchdir = os.path.relpath(srctreebase, srctree) + if relpatchdir == '.': + patchdir_params = {} + else: + patchdir_params = {'patchdir': relpatchdir} + + def srcuri_entry(fname): + if patchdir_params: + paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) + else: + paramstr = '' + return 'file://%s%s' % (basepath, paramstr) initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) if not initial_rev: @@ -1627,7 +1640,6 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil new_f = {} del_f = {} else: - srctreebase = workspace[recipename]['srctreebase'] upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) remove_files = [] @@ -1663,14 +1675,15 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil removedentries, remaining = _remove_file_entries( srcuri, remove_files) if removedentries or remaining: - remaining = ['file://' + os.path.basename(item) for + remaining = [srcuri_entry(os.path.basename(item)) for item in remaining] removevalues = {'SRC_URI': removedentries + remaining} appendfile, destpath = oe.recipeutils.bbappend_recipe( rd, appendlayerdir, files, wildcardver=wildcard_version, removevalues=removevalues, - redirect_output=dry_run_outdir) + redirect_output=dry_run_outdir, + params=[patchdir_params] * len(files)) else: logger.info('No patches or local source files needed updating') else: @@ -1694,7 +1707,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil # replace the entry in SRC_URI with our local version logger.info('Replacing remote patch %s with updated local version' % basepath) path = os.path.join(files_dir, basepath) - _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) + _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath)) updaterecipe = True else: logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) @@ -1708,7 +1721,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil os.path.join(files_dir, basepath), dry_run_outdir=dry_run_outdir, base_outdir=recipedir) - srcuri.append('file://%s' % basepath) + srcuri.append(srcuri_entry(basepath)) updaterecipe = True for basepath, path in new_p.items(): logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) @@ -1716,7 +1729,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil os.path.join(files_dir, basepath), dry_run_outdir=dry_run_outdir, base_outdir=recipedir) - srcuri.append('file://%s' % basepath) + srcuri.append(srcuri_entry(basepath)) updaterecipe = True # Update recipe, if needed if _remove_file_entries(srcuri, remove_files)[0]: From patchwork Wed Jul 13 01:41:25 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Eggleton X-Patchwork-Id: 10117 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 3E426C43334 for ; Wed, 13 Jul 2022 01:41:33 +0000 (UTC) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mx.groups.io with SMTP id smtpd.web12.17332.1657676490656014620 for ; Tue, 12 Jul 2022 18:41:30 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@linux.microsoft.com header.s=default header.b=UlgnqLsU; spf=pass (domain: linux.microsoft.com, ip: 13.77.154.182, mailfrom: pauleg@linux.microsoft.com) Received: by linux.microsoft.com (Postfix, from userid 1054) id 41DF4204DE8C; Tue, 12 Jul 2022 18:41:30 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 41DF4204DE8C DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1657676490; bh=pQEwse91fa3R5yJUAL66l4GQtvLnBM/2XINPu2k4Ms0=; h=From:To:Subject:Date:In-Reply-To:References:From; b=UlgnqLsUQ4lV/bOnXLRNxwHFxn9xdI1iiNxaRsac4OSFvEKEhjemj9slj/QB27SLY 8+DlGEDkFGcW0w6QVZNC9rygQJHI0RVhDpnpTMPMRnQuPu4kjJfnU6GMOYVeAJ3X5D i4d1+wAsmhvHPSyEjOqSBad7ZvSFCw0gTMulDDW4= From: Paul Eggleton To: openembedded-core@lists.openembedded.org Subject: [PATCH v2 3/3] oe-selftest: devtool: test modify git recipe building from a subdir Date: Tue, 12 Jul 2022 18:41:25 -0700 Message-Id: <8b0bd61a35b7803ce79ab670156da8a66b04dd15.1657676159.git.paul.eggleton@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: References: List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Wed, 13 Jul 2022 01:41:33 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/openembedded-core/message/167950 From: Paul Eggleton Add a test that verifies that devtool modify + devtool finish do the right thing on a recipe that fetches from git and sets S to point to a subdirectory of the source tree. We have a few examples among the core recipes, dos2unix is a convenient one so let's use that. (The test first verifies that that is still true in case the recipe is changed in future.) Signed-off-by: Paul Eggleton --- meta/lib/oeqa/selftest/cases/devtool.py | 114 +++++++++++++++++++++++++++----- 1 file changed, 97 insertions(+), 17 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py index ddf6c0c..34fc791 100644 --- a/meta/lib/oeqa/selftest/cases/devtool.py +++ b/meta/lib/oeqa/selftest/cases/devtool.py @@ -218,6 +218,34 @@ class DevtoolTestCase(OESelftestTestCase): filelist.append(' '.join(splitline)) return filelist + def _check_diff(self, diffoutput, addlines, removelines): + """Check output from 'git diff' matches expectation""" + remaining_addlines = addlines[:] + remaining_removelines = removelines[:] + for line in diffoutput.splitlines(): + if line.startswith('+++') or line.startswith('---'): + continue + elif line.startswith('+'): + matched = False + for item in addlines: + if re.match(item, line[1:].strip()): + matched = True + remaining_addlines.remove(item) + break + self.assertTrue(matched, 'Unexpected diff add line: %s' % line) + elif line.startswith('-'): + matched = False + for item in removelines: + if re.match(item, line[1:].strip()): + matched = True + remaining_removelines.remove(item) + break + self.assertTrue(matched, 'Unexpected diff remove line: %s' % line) + if remaining_addlines: + self.fail('Expected added lines not found: %s' % remaining_addlines) + if remaining_removelines: + self.fail('Expected removed lines not found: %s' % remaining_removelines) + class DevtoolBase(DevtoolTestCase): @@ -718,6 +746,7 @@ class DevtoolModifyTests(DevtoolBase): self.assertTrue(bbclassextended, 'None of these recipes are BBCLASSEXTENDed to native - need to adjust testrecipes list: %s' % ', '.join(testrecipes)) self.assertTrue(inheritnative, 'None of these recipes do "inherit native" - need to adjust testrecipes list: %s' % ', '.join(testrecipes)) + def test_devtool_modify_localfiles_only(self): # Check preconditions testrecipe = 'base-files' @@ -930,23 +959,7 @@ class DevtoolUpdateTests(DevtoolBase): srcurilines[0] = 'SRC_URI = "' + srcurilines[0] srcurilines.append('"') removelines = ['SRCREV = ".*"'] + srcurilines - for line in result.output.splitlines(): - if line.startswith('+++') or line.startswith('---'): - continue - elif line.startswith('+'): - matched = False - for item in addlines: - if re.match(item, line[1:].strip()): - matched = True - break - self.assertTrue(matched, 'Unexpected diff add line: %s' % line) - elif line.startswith('-'): - matched = False - for item in removelines: - if re.match(item, line[1:].strip()): - matched = True - break - self.assertTrue(matched, 'Unexpected diff remove line: %s' % line) + self._check_diff(result.output, addlines, removelines) # Now try with auto mode runCmd('cd %s; git checkout %s %s' % (os.path.dirname(recipefile), testrecipe, os.path.basename(recipefile))) result = runCmd('devtool update-recipe %s' % testrecipe) @@ -1316,6 +1329,73 @@ class DevtoolUpdateTests(DevtoolBase): expected_status = [] self._check_repo_status(os.path.dirname(recipefile), expected_status) + def test_devtool_finish_modify_git_subdir(self): + # Check preconditions + testrecipe = 'dos2unix' + bb_vars = get_bb_vars(['SRC_URI', 'S', 'WORKDIR', 'FILE'], testrecipe) + self.assertIn('git://', bb_vars['SRC_URI'], 'This test expects the %s recipe to be a git recipe' % testrecipe) + workdir_git = '%s/git/' % bb_vars['WORKDIR'] + if not bb_vars['S'].startswith(workdir_git): + self.fail('This test expects the %s recipe to be building from a subdirectory of the git repo' % testrecipe) + subdir = bb_vars['S'].split(workdir_git, 1)[1] + # Clean up anything in the workdir/sysroot/sstate cache + bitbake('%s -c cleansstate' % testrecipe) + # Try modifying a recipe + tempdir = tempfile.mkdtemp(prefix='devtoolqa') + self.track_for_cleanup(tempdir) + self.track_for_cleanup(self.workspacedir) + self.add_command_to_tearDown('bitbake -c clean %s' % testrecipe) + self.add_command_to_tearDown('bitbake-layers remove-layer */workspace') + result = runCmd('devtool modify %s -x %s' % (testrecipe, tempdir)) + testsrcfile = os.path.join(tempdir, subdir, 'dos2unix.c') + self.assertExists(testsrcfile, 'Extracted source could not be found') + self.assertExists(os.path.join(self.workspacedir, 'conf', 'layer.conf'), 'Workspace directory not created. devtool output: %s' % result.output) + self.assertNotExists(os.path.join(tempdir, subdir, '.git'), 'Subdirectory has been initialised as a git repo') + # Check git repo + self._check_src_repo(tempdir) + # Modify file + runCmd("sed -i '1s:^:/* Add a comment */\\n:' %s" % testsrcfile) + result = runCmd('git commit -a -m "Add a comment"', cwd=tempdir) + # Now try updating original recipe + recipefile = bb_vars['FILE'] + recipedir = os.path.dirname(recipefile) + self.add_command_to_tearDown('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe)) + result = runCmd('devtool update-recipe %s' % testrecipe) + expected_status = [(' M', '.*/%s$' % os.path.basename(recipefile)), + ('??', '.*/%s/%s/$' % (testrecipe, testrecipe))] + self._check_repo_status(os.path.dirname(recipefile), expected_status) + result = runCmd('git diff %s' % os.path.basename(recipefile), cwd=os.path.dirname(recipefile)) + removelines = ['SRC_URI = "git://.*"'] + addlines = [ + 'SRC_URI = "git://.* \\\\', + 'file://0001-Add-a-comment.patch;patchdir=.. \\\\', + '"' + ] + self._check_diff(result.output, addlines, removelines) + # Put things back so we can run devtool finish on a different layer + runCmd('cd %s; rm -f %s/*.patch; git checkout .' % (recipedir, testrecipe)) + # Run devtool finish + res = re.search('recipes-.*', recipedir) + self.assertTrue(res, 'Unable to find recipe subdirectory') + recipesubdir = res[0] + self.add_command_to_tearDown('rm -rf %s' % os.path.join(self.testlayer_path, recipesubdir)) + result = runCmd('devtool finish %s meta-selftest' % testrecipe) + # Check bbappend file contents + appendfn = os.path.join(self.testlayer_path, recipesubdir, '%s_%%.bbappend' % testrecipe) + with open(appendfn, 'r') as f: + appendlines = f.readlines() + expected_appendlines = [ + 'FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', + '\n', + 'SRC_URI += "file://0001-Add-a-comment.patch;patchdir=.."\n', + '\n' + ] + self.assertEqual(appendlines, expected_appendlines) + self.assertExists(os.path.join(os.path.dirname(appendfn), testrecipe, '0001-Add-a-comment.patch')) + # Try building + bitbake('%s -c patch' % testrecipe) + + class DevtoolExtractTests(DevtoolBase): def test_devtool_extract(self):