| Message ID | 20230125083127.53988-1-lrannou@baylibre.com |
|---|---|
| State | New |
| Headers | show |
| Series | oeqa/selftest/locales: Create selftest for locales generation | expand |
On Wed, 2023-01-25 at 09:31 +0100, Louis Rannou wrote: > From: Louis Rannou <lrannou@baylibre.com> > > [YOCTO #9070] > > Add a new selftest to validate the locales generation. This selftest builds a > complete target with GLIBC_GENERATE_LOCALES, IMAGE_LINGUAS, > ENABLE_BINARY_LOCALE_GENERATION set. The locales are tested through a ssh > connection in three manners: checks if the locales exist in the locale-archive, > if it does exist for the glibc and if it can be generated. > > Signed-off-by: Louis Rannou <lrannou@baylibre.com> > --- > meta/lib/oeqa/selftest/cases/locales.py | 54 +++++++++++++++++++++++++ > 1 file changed, 54 insertions(+) > create mode 100644 meta/lib/oeqa/selftest/cases/locales.py > > diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py > new file mode 100644 > index 0000000000..089ca594a1 > --- /dev/null > +++ b/meta/lib/oeqa/selftest/cases/locales.py > @@ -0,0 +1,54 @@ > +# > +# SPDX-License-Identifier: MIT > +# > + > +from oeqa.selftest.case import OESelftestTestCase > +from oeqa.core.decorator import OETestTag > +from oeqa.utils.commands import bitbake, runqemu > +from oeqa.utils.sshcontrol import SSHControl > + > +class LocalesTest(OESelftestTestCase): > + > + @OETestTag("runqemu") > + def test_locales(self): > + """ > + Summary: Test the locales through a ssh connection > + Expected: 1. Check the locale exist in the locale-archive > + 2. Check it the locale exist for the glibc > + 3. Check the locale can be generated > + Product: oe-core > + Author: Louis Rannou <lrannou@baylibre.com> > + AutomatedBy: Louis Rannou <lrannou@baylibre.com> > + """ > + > + features = [] > + features.append('EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password allow-root-login"') > + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') > + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') > + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') > + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') > + self.write_config("\n".join(features)) > + > + # Build a core-image-minimal > + bitbake('core-image-minimal') > + > + with runqemu("core-image-minimal") as qemu: > + # Attempt to ssh with each user into qemu with empty password > + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user='root') Thanks for this, in general this looks good and it would be great to get a test for this enabled so I'm very happy to see the patch. I did have one question about the above line. I think you can just use qemu.run() to run commands over ssh and I suspect the above might create a second ssh connection which might then have lifecycle issues. Did you try qemu.run() and have issues? I noticed a lot of our tests use run_serial() which I hadn't realised until I looked! Cheers, Richard > + cmd = "locale -a" > + status, output = ssh.run(cmd) > + # output must includes fr_FR or fr_FR.UTF-8 > + self.assertEqual(status, 0, msg='locale -a test failed: %s' % output) > + self.assertIn("fr_FR", output, msg='Incorrect output: %s' % output) > + > + cmd = "localedef --list-archive -v | grep fr_FR.utf8" > + status, output = ssh.run(cmd) > + # output must includes fr_FR.utf8 > + self.assertEqual(status, 0, msg='localedef test failed: %s' % output) > + > + cmd = "LANG=fr_FR.UTF-8 locale" > + status, output = ssh.run(cmd) > + # output must includes fr_FR.UTF-8 > + self.assertEqual(status, 0, msg='locale test failed: %s' % output) > + self.assertIn("LANG=fr_FR.UTF-8", output, > + msg='Incorrect output: %s' % output)
On 26/01/2023 11:02, Richard Purdie wrote: > On Wed, 2023-01-25 at 09:31 +0100, Louis Rannou wrote: >> From: Louis Rannou <lrannou@baylibre.com> >> >> [YOCTO #9070] >> >> Add a new selftest to validate the locales generation. This selftest builds a >> complete target with GLIBC_GENERATE_LOCALES, IMAGE_LINGUAS, >> ENABLE_BINARY_LOCALE_GENERATION set. The locales are tested through a ssh >> connection in three manners: checks if the locales exist in the locale-archive, >> if it does exist for the glibc and if it can be generated. >> >> Signed-off-by: Louis Rannou <lrannou@baylibre.com> >> --- >> meta/lib/oeqa/selftest/cases/locales.py | 54 +++++++++++++++++++++++++ >> 1 file changed, 54 insertions(+) >> create mode 100644 meta/lib/oeqa/selftest/cases/locales.py >> >> diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py >> new file mode 100644 >> index 0000000000..089ca594a1 >> --- /dev/null >> +++ b/meta/lib/oeqa/selftest/cases/locales.py >> @@ -0,0 +1,54 @@ >> +# >> +# SPDX-License-Identifier: MIT >> +# >> + >> +from oeqa.selftest.case import OESelftestTestCase >> +from oeqa.core.decorator import OETestTag >> +from oeqa.utils.commands import bitbake, runqemu >> +from oeqa.utils.sshcontrol import SSHControl >> + >> +class LocalesTest(OESelftestTestCase): >> + >> + @OETestTag("runqemu") >> + def test_locales(self): >> + """ >> + Summary: Test the locales through a ssh connection >> + Expected: 1. Check the locale exist in the locale-archive >> + 2. Check it the locale exist for the glibc >> + 3. Check the locale can be generated >> + Product: oe-core >> + Author: Louis Rannou <lrannou@baylibre.com> >> + AutomatedBy: Louis Rannou <lrannou@baylibre.com> >> + """ >> + >> + features = [] >> + features.append('EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password allow-root-login"') >> + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') >> + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') >> + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') >> + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') >> + self.write_config("\n".join(features)) >> + >> + # Build a core-image-minimal >> + bitbake('core-image-minimal') >> + >> + with runqemu("core-image-minimal") as qemu: >> + # Attempt to ssh with each user into qemu with empty password >> + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user='root') > > Thanks for this, in general this looks good and it would be great to > get a test for this enabled so I'm very happy to see the patch. > > I did have one question about the above line. I think you can just use > qemu.run() to run commands over ssh and I suspect the above might > create a second ssh connection which might then have lifecycle issues. > Did you try qemu.run() and have issues? I noticed a lot of our tests > use run_serial() which I hadn't realised until I looked! Hello, I haven't tried qemu as the bugzilla said to use a ssh connection. But I guess this was different in 2013. I agree, it might be better to avoid the ssh connection. Also, the bugzilla asks to make a primary test with ENABLE_BINARY_LOCALE_GENERATION="0" to check the locales are not gener. This means a complete regeneration of the image, so I thought it is not very relevant. Is this fine ? Thanks, Louis > > Cheers, > > Richard > >> + cmd = "locale -a" >> + status, output = ssh.run(cmd) >> + # output must includes fr_FR or fr_FR.UTF-8 >> + self.assertEqual(status, 0, msg='locale -a test failed: %s' % output) >> + self.assertIn("fr_FR", output, msg='Incorrect output: %s' % output) >> + >> + cmd = "localedef --list-archive -v | grep fr_FR.utf8" >> + status, output = ssh.run(cmd) >> + # output must includes fr_FR.utf8 >> + self.assertEqual(status, 0, msg='localedef test failed: %s' % output) >> + >> + cmd = "LANG=fr_FR.UTF-8 locale" >> + status, output = ssh.run(cmd) >> + # output must includes fr_FR.UTF-8 >> + self.assertEqual(status, 0, msg='locale test failed: %s' % output) >> + self.assertIn("LANG=fr_FR.UTF-8", output, >> + msg='Incorrect output: %s' % output) >
On Thu, 2023-01-26 at 11:09 +0100, Louis Rannou wrote: > On 26/01/2023 11:02, Richard Purdie wrote: > > On Wed, 2023-01-25 at 09:31 +0100, Louis Rannou wrote: > > > From: Louis Rannou <lrannou@baylibre.com> > > > > > > [YOCTO #9070] > > > > > > Add a new selftest to validate the locales generation. This selftest builds a > > > complete target with GLIBC_GENERATE_LOCALES, IMAGE_LINGUAS, > > > ENABLE_BINARY_LOCALE_GENERATION set. The locales are tested through a ssh > > > connection in three manners: checks if the locales exist in the locale-archive, > > > if it does exist for the glibc and if it can be generated. > > > > > > Signed-off-by: Louis Rannou <lrannou@baylibre.com> > > > --- > > > meta/lib/oeqa/selftest/cases/locales.py | 54 +++++++++++++++++++++++++ > > > 1 file changed, 54 insertions(+) > > > create mode 100644 meta/lib/oeqa/selftest/cases/locales.py > > > > > > diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py > > > new file mode 100644 > > > index 0000000000..089ca594a1 > > > --- /dev/null > > > +++ b/meta/lib/oeqa/selftest/cases/locales.py > > > @@ -0,0 +1,54 @@ > > > +# > > > +# SPDX-License-Identifier: MIT > > > +# > > > + > > > +from oeqa.selftest.case import OESelftestTestCase > > > +from oeqa.core.decorator import OETestTag > > > +from oeqa.utils.commands import bitbake, runqemu > > > +from oeqa.utils.sshcontrol import SSHControl > > > + > > > +class LocalesTest(OESelftestTestCase): > > > + > > > + @OETestTag("runqemu") > > > + def test_locales(self): > > > + """ > > > + Summary: Test the locales through a ssh connection > > > + Expected: 1. Check the locale exist in the locale-archive > > > + 2. Check it the locale exist for the glibc > > > + 3. Check the locale can be generated > > > + Product: oe-core > > > + Author: Louis Rannou <lrannou@baylibre.com> > > > + AutomatedBy: Louis Rannou <lrannou@baylibre.com> > > > + """ > > > + > > > + features = [] > > > + features.append('EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password allow-root-login"') > > > + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') > > > + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') > > > + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') > > > + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') > > > + self.write_config("\n".join(features)) > > > + > > > + # Build a core-image-minimal > > > + bitbake('core-image-minimal') > > > + > > > + with runqemu("core-image-minimal") as qemu: > > > + # Attempt to ssh with each user into qemu with empty password > > > + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user='root') > > > > Thanks for this, in general this looks good and it would be great to > > get a test for this enabled so I'm very happy to see the patch. > > > > I did have one question about the above line. I think you can just use > > qemu.run() to run commands over ssh and I suspect the above might > > create a second ssh connection which might then have lifecycle issues. > > Did you try qemu.run() and have issues? I noticed a lot of our tests > > use run_serial() which I hadn't realised until I looked! > Hello, I haven't tried qemu as the bugzilla said to use a ssh > connection. But I guess this was different in 2013. I agree, it might be > better to avoid the ssh connection. To be clear, qemu.run() should run the command over an ssh connection to qemu if I remember the code paths correctly. It is just abstracted in the runqemu code so the test doesn't need to worry about it. > Also, the bugzilla asks to make a primary test with > ENABLE_BINARY_LOCALE_GENERATION="0" to check the locales are not gener. > This means a complete regeneration of the image, so I thought it is not > very relevant. Is this fine ? Ideally we'd have two tests, one for binary locales enabled and one with it not enabled. That will mean a little more build time but would completely test the different options so is probably worth doing. I'd be happy for *any* automated locale tests at this point though! We shouldn't close the bug until we have both tests. Cheers, Richard
On 26/01/2023 11:15, Richard Purdie wrote: > On Thu, 2023-01-26 at 11:09 +0100, Louis Rannou wrote: >> On 26/01/2023 11:02, Richard Purdie wrote: >>> On Wed, 2023-01-25 at 09:31 +0100, Louis Rannou wrote: >>>> From: Louis Rannou <lrannou@baylibre.com> >>>> >>>> [YOCTO #9070] >>>> >>>> Add a new selftest to validate the locales generation. This selftest builds a >>>> complete target with GLIBC_GENERATE_LOCALES, IMAGE_LINGUAS, >>>> ENABLE_BINARY_LOCALE_GENERATION set. The locales are tested through a ssh >>>> connection in three manners: checks if the locales exist in the locale-archive, >>>> if it does exist for the glibc and if it can be generated. >>>> >>>> Signed-off-by: Louis Rannou <lrannou@baylibre.com> >>>> --- >>>> meta/lib/oeqa/selftest/cases/locales.py | 54 +++++++++++++++++++++++++ >>>> 1 file changed, 54 insertions(+) >>>> create mode 100644 meta/lib/oeqa/selftest/cases/locales.py >>>> >>>> diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py >>>> new file mode 100644 >>>> index 0000000000..089ca594a1 >>>> --- /dev/null >>>> +++ b/meta/lib/oeqa/selftest/cases/locales.py >>>> @@ -0,0 +1,54 @@ >>>> +# >>>> +# SPDX-License-Identifier: MIT >>>> +# >>>> + >>>> +from oeqa.selftest.case import OESelftestTestCase >>>> +from oeqa.core.decorator import OETestTag >>>> +from oeqa.utils.commands import bitbake, runqemu >>>> +from oeqa.utils.sshcontrol import SSHControl >>>> + >>>> +class LocalesTest(OESelftestTestCase): >>>> + >>>> + @OETestTag("runqemu") >>>> + def test_locales(self): >>>> + """ >>>> + Summary: Test the locales through a ssh connection >>>> + Expected: 1. Check the locale exist in the locale-archive >>>> + 2. Check it the locale exist for the glibc >>>> + 3. Check the locale can be generated >>>> + Product: oe-core >>>> + Author: Louis Rannou <lrannou@baylibre.com> >>>> + AutomatedBy: Louis Rannou <lrannou@baylibre.com> >>>> + """ >>>> + >>>> + features = [] >>>> + features.append('EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password allow-root-login"') >>>> + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') >>>> + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') >>>> + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') >>>> + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') >>>> + self.write_config("\n".join(features)) >>>> + >>>> + # Build a core-image-minimal >>>> + bitbake('core-image-minimal') >>>> + >>>> + with runqemu("core-image-minimal") as qemu: >>>> + # Attempt to ssh with each user into qemu with empty password >>>> + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user='root') >>> >>> Thanks for this, in general this looks good and it would be great to >>> get a test for this enabled so I'm very happy to see the patch. >>> >>> I did have one question about the above line. I think you can just use >>> qemu.run() to run commands over ssh and I suspect the above might >>> create a second ssh connection which might then have lifecycle issues. >>> Did you try qemu.run() and have issues? I noticed a lot of our tests >>> use run_serial() which I hadn't realised until I looked! >> Hello, I haven't tried qemu as the bugzilla said to use a ssh >> connection. But I guess this was different in 2013. I agree, it might be >> better to avoid the ssh connection. I just noticed that qemu_serial returns 1 on success and 0 on failure while the ssh command returns 0 and -N on failure. > > To be clear, qemu.run() should run the command over an ssh connection > to qemu if I remember the code paths correctly. It is just abstracted > in the runqemu code so the test doesn't need to worry about it. Indeed, I found this is correct. > >> Also, the bugzilla asks to make a primary test with >> ENABLE_BINARY_LOCALE_GENERATION="0" to check the locales are not gener. >> This means a complete regeneration of the image, so I thought it is not >> very relevant. Is this fine ? > > Ideally we'd have two tests, one for binary locales enabled and one > with it not enabled. That will mean a little more build time but would > completely test the different options so is probably worth doing. I'd > be happy for *any* automated locale tests at this point though! Ok, let's do that ! Thank you, Louis
On Fri, 2023-01-27 at 14:47 +0100, Louis Rannou wrote: > > On 26/01/2023 11:15, Richard Purdie wrote: > > On Thu, 2023-01-26 at 11:09 +0100, Louis Rannou wrote: > > > On 26/01/2023 11:02, Richard Purdie wrote: > > > > On Wed, 2023-01-25 at 09:31 +0100, Louis Rannou wrote: > > > > > From: Louis Rannou <lrannou@baylibre.com> > > > > > > > > > > [YOCTO #9070] > > > > > > > > > > Add a new selftest to validate the locales generation. This selftest builds a > > > > > complete target with GLIBC_GENERATE_LOCALES, IMAGE_LINGUAS, > > > > > ENABLE_BINARY_LOCALE_GENERATION set. The locales are tested through a ssh > > > > > connection in three manners: checks if the locales exist in the locale-archive, > > > > > if it does exist for the glibc and if it can be generated. > > > > > > > > > > Signed-off-by: Louis Rannou <lrannou@baylibre.com> > > > > > --- > > > > > meta/lib/oeqa/selftest/cases/locales.py | 54 +++++++++++++++++++++++++ > > > > > 1 file changed, 54 insertions(+) > > > > > create mode 100644 meta/lib/oeqa/selftest/cases/locales.py > > > > > > > > > > diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py > > > > > new file mode 100644 > > > > > index 0000000000..089ca594a1 > > > > > --- /dev/null > > > > > +++ b/meta/lib/oeqa/selftest/cases/locales.py > > > > > @@ -0,0 +1,54 @@ > > > > > +# > > > > > +# SPDX-License-Identifier: MIT > > > > > +# > > > > > + > > > > > +from oeqa.selftest.case import OESelftestTestCase > > > > > +from oeqa.core.decorator import OETestTag > > > > > +from oeqa.utils.commands import bitbake, runqemu > > > > > +from oeqa.utils.sshcontrol import SSHControl > > > > > + > > > > > +class LocalesTest(OESelftestTestCase): > > > > > + > > > > > + @OETestTag("runqemu") > > > > > + def test_locales(self): > > > > > + """ > > > > > + Summary: Test the locales through a ssh connection > > > > > + Expected: 1. Check the locale exist in the locale-archive > > > > > + 2. Check it the locale exist for the glibc > > > > > + 3. Check the locale can be generated > > > > > + Product: oe-core > > > > > + Author: Louis Rannou <lrannou@baylibre.com> > > > > > + AutomatedBy: Louis Rannou <lrannou@baylibre.com> > > > > > + """ > > > > > + > > > > > + features = [] > > > > > + features.append('EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password allow-root-login"') > > > > > + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') > > > > > + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') > > > > > + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') > > > > > + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') > > > > > + self.write_config("\n".join(features)) > > > > > + > > > > > + # Build a core-image-minimal > > > > > + bitbake('core-image-minimal') > > > > > + > > > > > + with runqemu("core-image-minimal") as qemu: > > > > > + # Attempt to ssh with each user into qemu with empty password > > > > > + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user='root') > > > > > > > > Thanks for this, in general this looks good and it would be great to > > > > get a test for this enabled so I'm very happy to see the patch. > > > > > > > > I did have one question about the above line. I think you can just use > > > > qemu.run() to run commands over ssh and I suspect the above might > > > > create a second ssh connection which might then have lifecycle issues. > > > > Did you try qemu.run() and have issues? I noticed a lot of our tests > > > > use run_serial() which I hadn't realised until I looked! > > > Hello, I haven't tried qemu as the bugzilla said to use a ssh > > > connection. But I guess this was different in 2013. I agree, it might be > > > better to avoid the ssh connection. > > I just noticed that qemu_serial returns 1 on success and 0 on failure > while the ssh command returns 0 and -N on failure. That is horrible. We should file a bug so we remember to improve that! Since it is python, they probably should raise an exception if they fail instead or a return value. Cheers, Richard
diff --git a/meta/lib/oeqa/selftest/cases/locales.py b/meta/lib/oeqa/selftest/cases/locales.py new file mode 100644 index 0000000000..089ca594a1 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/locales.py @@ -0,0 +1,54 @@ +# +# SPDX-License-Identifier: MIT +# + +from oeqa.selftest.case import OESelftestTestCase +from oeqa.core.decorator import OETestTag +from oeqa.utils.commands import bitbake, runqemu +from oeqa.utils.sshcontrol import SSHControl + +class LocalesTest(OESelftestTestCase): + + @OETestTag("runqemu") + def test_locales(self): + """ + Summary: Test the locales through a ssh connection + Expected: 1. Check the locale exist in the locale-archive + 2. Check it the locale exist for the glibc + 3. Check the locale can be generated + Product: oe-core + Author: Louis Rannou <lrannou@baylibre.com> + AutomatedBy: Louis Rannou <lrannou@baylibre.com> + """ + + features = [] + features.append('EXTRA_IMAGE_FEATURES = "ssh-server-openssh empty-root-password allow-empty-password allow-root-login"') + features.append('IMAGE_INSTALL:append = " glibc-utils localedef"') + features.append('GLIBC_GENERATE_LOCALES = "en_US.UTF-8 fr_FR.UTF-8"') + features.append('IMAGE_LINGUAS:append = " en-us fr-fr"') + features.append('ENABLE_BINARY_LOCALE_GENERATION = "1"') + self.write_config("\n".join(features)) + + # Build a core-image-minimal + bitbake('core-image-minimal') + + with runqemu("core-image-minimal") as qemu: + # Attempt to ssh with each user into qemu with empty password + ssh = SSHControl(ip=qemu.ip, logfile=qemu.sshlog, user='root') + cmd = "locale -a" + status, output = ssh.run(cmd) + # output must includes fr_FR or fr_FR.UTF-8 + self.assertEqual(status, 0, msg='locale -a test failed: %s' % output) + self.assertIn("fr_FR", output, msg='Incorrect output: %s' % output) + + cmd = "localedef --list-archive -v | grep fr_FR.utf8" + status, output = ssh.run(cmd) + # output must includes fr_FR.utf8 + self.assertEqual(status, 0, msg='localedef test failed: %s' % output) + + cmd = "LANG=fr_FR.UTF-8 locale" + status, output = ssh.run(cmd) + # output must includes fr_FR.UTF-8 + self.assertEqual(status, 0, msg='locale test failed: %s' % output) + self.assertIn("LANG=fr_FR.UTF-8", output, + msg='Incorrect output: %s' % output)