Patchwork [bitbake-devel,7/8] command.py: add new command to get the CPU info

login
register
mail settings
Submitter Dongxiao Xu
Date Jan. 11, 2012, 3:03 a.m.
Message ID <d628d53679aa12f5803d0bbf30aa21ed3bc4779d.1326249301.git.dongxiao.xu@intel.com>
Download mbox | patch
Permalink /patch/19017/
State New
Headers show

Comments

Dongxiao Xu - Jan. 11, 2012, 3:03 a.m.
Add new API in command.py to get the CPU core and threads information
in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
variables.

Signed-off-by: Shane Wang <shane.wang@intel.com>
---
 bitbake/lib/bb/helper.py |   39 +++++++++++++++++++++++++++++++++++++++
 lib/bb/command.py        |   12 ++++++++++++
 2 files changed, 51 insertions(+), 0 deletions(-)
 create mode 100644 bitbake/lib/bb/helper.py
Richard Purdie - Jan. 11, 2012, 11:34 a.m.
On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
> Add new API in command.py to get the CPU core and threads information
> in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
> variables.
> 
> Signed-off-by: Shane Wang <shane.wang@intel.com>
> ---
>  bitbake/lib/bb/helper.py |   39 +++++++++++++++++++++++++++++++++++++++
>  lib/bb/command.py        |   12 ++++++++++++
>  2 files changed, 51 insertions(+), 0 deletions(-)
>  create mode 100644 bitbake/lib/bb/helper.py
> 
> diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py
> new file mode 100644
> index 0000000..291158b
> --- /dev/null
> +++ b/bitbake/lib/bb/helper.py
> @@ -0,0 +1,39 @@
> +#
> +# Helper for BitBake Graphical GTK User Interface
> +#
> +# Copyright (C) 2011        Intel Corporation
> +#
> +# Authored by Shane Wang <shane.wang@intel.com>
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License version 2 as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License along
> +# with this program; if not, write to the Free Software Foundation, Inc.,
> +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +import os
> +
> +class CpuInfo(object):
> +
> +    coefficient = 4

This is setting off alarm bells...

> +    @classmethod
> +    def getNumOfCpus(cls):
> +        pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
> +        num = len(pfile.readlines())
> +        return num

You can get this number with something like:

import multiprocessing
multiprocessing.cpu_count()

> +    @classmethod
> +    def getNumOfCpuCores(cls):
> +        pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2')
> +        contents = pfile.readlines()
> +        num = int(contents[0])
> +        return num

I'm curious what you're using the number of cores to do? It doesn't seem
used by your code?


> diff --git a/lib/bb/command.py b/lib/bb/command.py
> index 05555c5..eaf8236 100644
> --- a/lib/bb/command.py
> +++ b/lib/bb/command.py
> @@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
>  
>  import bb.event
>  import bb.cooker
> +import bb.helper
>  
>  class CommandCompleted(bb.event.Event):
>      pass
> @@ -173,6 +174,17 @@ class CommandsSync:
>          """
>          command.cooker.reset()
>  
> +    def getDefaultNumOfThreads(self, command, params):
> +        """
> +        Get the default number of threads on the server = number of CPUs
> +        """
> +        return bb.helper.CpuInfo.getNumOfCpus()
> +
> +    def getMaxNumOfThreads(self, command, params):
> +        """
> +        Get the max number of threads that the server can tolerate
> +        """
> +        return bb.helper.CpuInfo.getNumOfCpus() * bb.helper.CpuInfo.coefficient
>  

I can understand needing to query the number of cpus but this last
function seems rather arbitrary. If you want to do bounds checking, I'd
suggest just adding the factor of 4 into the UI. I'm still not convinced
we should be setting any value for this though, or doing any bounds
checking on the value.


My only other comment would be to use bb.utils instead of creating a
bb.helper.

Cheers,

Richard
Shane Wang - Jan. 11, 2012, 11:52 a.m.
Richard Purdie wrote on 2012-01-11:

> On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
>> Add new API in command.py to get the CPU core and threads information
>> in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
>> variables.
>> 
>> Signed-off-by: Shane Wang <shane.wang@intel.com>
>> ---
>>  bitbake/lib/bb/helper.py |   39
>>  +++++++++++++++++++++++++++++++++++++++ lib/bb/command.py        |  
>>  12 ++++++++++++ 2 files changed, 51 insertions(+), 0 deletions(-)
>>  create mode 100644 bitbake/lib/bb/helper.py
>> diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py new
>> file mode 100644 index 0000000..291158b --- /dev/null +++
>> b/bitbake/lib/bb/helper.py @@ -0,0 +1,39 @@ +# +# Helper for BitBake
>> Graphical GTK User Interface +# +# Copyright (C) 2011        Intel
>> Corporation +# +# Authored by Shane Wang <shane.wang@intel.com> +# +#
>> This program is free software; you can redistribute it and/or modify +#
>> it under the terms of the GNU General Public License version 2 as +#
>> published by the Free Software Foundation. +# +# This program is
>> distributed in the hope that it will be useful, +# but WITHOUT ANY
>> WARRANTY; without even the implied warranty of +# MERCHANTABILITY or
>> FITNESS FOR A PARTICULAR PURPOSE.  See the +# GNU General Public
>> License for more details. +# +# You should have received a copy of the
>> GNU General Public License along +# with this program; if not, write to
>> the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor,
>> Boston, MA 02110-1301 USA. + +import os + +class CpuInfo(object): + +  
>>  coefficient = 4
> 
> This is setting off alarm bells...
> 
>> +    @classmethod
>> +    def getNumOfCpus(cls):
>> +        pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
>> +        num = len(pfile.readlines())
>> +        return num
> 
> You can get this number with something like:
> 
> import multiprocessing
> multiprocessing.cpu_count()
Good to see this works, will change that.

> 
>> +    @classmethod +    def getNumOfCpuCores(cls): +        pfile =
>> os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2') +       
>> contents = pfile.readlines() +        num = int(contents[0]) +       
>> return num
> 
> I'm curious what you're using the number of cores to do? It doesn't seem
> used by your code?
> 
> 
>> diff --git a/lib/bb/command.py b/lib/bb/command.py
>> index 05555c5..eaf8236 100644
>> --- a/lib/bb/command.py
>> +++ b/lib/bb/command.py
>> @@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
>> 
>>  import bb.event
>>  import bb.cooker
>> +import bb.helper
>> 
>>  class CommandCompleted(bb.event.Event):
>>      pass
>> @@ -173,6 +174,17 @@ class CommandsSync:
>>          """
>>          command.cooker.reset()
>> +    def getDefaultNumOfThreads(self, command, params): +        """ + 
>>       Get the default number of threads on the server = number of CPUs
>> +        """ +        return bb.helper.CpuInfo.getNumOfCpus() + +   
>> def getMaxNumOfThreads(self, command, params): +        """ +       
>> Get the max number of threads that the server can tolerate +        """
>> +        return bb.helper.CpuInfo.getNumOfCpus() *
>> bb.helper.CpuInfo.coefficient
>> 
> 
> I can understand needing to query the number of cpus but this last
> function seems rather arbitrary. If you want to do bounds checking, I'd
> suggest just adding the factor of 4 into the UI.
OK

> I'm still not convinced
> we should be setting any value for this though, or doing any bounds
> checking on the value.
Which value? We hope when the UI starts up, the UI can take advantage of cpus for build and set it as default.
But users can change between 0 ~ max num of threads.

> 
> 
> My only other comment would be to use bb.utils instead of creating a
> bb.helper.
> 
> Cheers,
> 
> Richard
> 
> 
> 
> _______________________________________________
> bitbake-devel mailing list
> bitbake-devel@lists.openembedded.org
> http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel
Shane Wang - Jan. 11, 2012, 12:02 p.m.
Wang, Shane wrote on 2012-01-11:
>> I'm still not convinced
>> we should be setting any value for this though, or doing any bounds
>> checking on the value.
> Which value? We hope when the UI starts up, the UI can take advantage of
> cpus for build and set it as default.
> But users can change between 0 ~ max num of threads.

Do you mean the max is not necessary and we can't get accurate max actually?

--
Shane
Richard Purdie - Jan. 11, 2012, 1:24 p.m.
On Wed, 2012-01-11 at 11:52 +0000, Wang, Shane wrote:
> Richard Purdie wrote on 2012-01-11:
> 
> > On Wed, 2012-01-11 at 11:03 +0800, Dongxiao Xu wrote:
> >> Add new API in command.py to get the CPU core and threads information
> >> in order to set the appropriate BB_NUMBER_THREADS and PARALLEL_MAKE
> >> variables.
> >> 
> >> Signed-off-by: Shane Wang <shane.wang@intel.com>
> >> ---
> >>  bitbake/lib/bb/helper.py |   39
> >>  +++++++++++++++++++++++++++++++++++++++ lib/bb/command.py        |  
> >>  12 ++++++++++++ 2 files changed, 51 insertions(+), 0 deletions(-)
> >>  create mode 100644 bitbake/lib/bb/helper.py
> >> diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py new
> >> file mode 100644 index 0000000..291158b --- /dev/null +++
> >> b/bitbake/lib/bb/helper.py @@ -0,0 +1,39 @@ +# +# Helper for BitBake
> >> Graphical GTK User Interface +# +# Copyright (C) 2011        Intel
> >> Corporation +# +# Authored by Shane Wang <shane.wang@intel.com> +# +#
> >> This program is free software; you can redistribute it and/or modify +#
> >> it under the terms of the GNU General Public License version 2 as +#
> >> published by the Free Software Foundation. +# +# This program is
> >> distributed in the hope that it will be useful, +# but WITHOUT ANY
> >> WARRANTY; without even the implied warranty of +# MERCHANTABILITY or
> >> FITNESS FOR A PARTICULAR PURPOSE.  See the +# GNU General Public
> >> License for more details. +# +# You should have received a copy of the
> >> GNU General Public License along +# with this program; if not, write to
> >> the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor,
> >> Boston, MA 02110-1301 USA. + +import os + +class CpuInfo(object): + +  
> >>  coefficient = 4
> > 
> > This is setting off alarm bells...
> > 
> >> +    @classmethod
> >> +    def getNumOfCpus(cls):
> >> +        pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
> >> +        num = len(pfile.readlines())
> >> +        return num
> > 
> > You can get this number with something like:
> > 
> > import multiprocessing
> > multiprocessing.cpu_count()
> Good to see this works, will change that.
> 
> > 
> >> +    @classmethod +    def getNumOfCpuCores(cls): +        pfile =
> >> os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2') +       
> >> contents = pfile.readlines() +        num = int(contents[0]) +       
> >> return num
> > 
> > I'm curious what you're using the number of cores to do? It doesn't seem
> > used by your code?
> > 
> > 
> >> diff --git a/lib/bb/command.py b/lib/bb/command.py
> >> index 05555c5..eaf8236 100644
> >> --- a/lib/bb/command.py
> >> +++ b/lib/bb/command.py
> >> @@ -30,6 +30,7 @@ Commands are queued in a CommandQueue
> >> 
> >>  import bb.event
> >>  import bb.cooker
> >> +import bb.helper
> >> 
> >>  class CommandCompleted(bb.event.Event):
> >>      pass
> >> @@ -173,6 +174,17 @@ class CommandsSync:
> >>          """
> >>          command.cooker.reset()
> >> +    def getDefaultNumOfThreads(self, command, params): +        """ + 
> >>       Get the default number of threads on the server = number of CPUs
> >> +        """ +        return bb.helper.CpuInfo.getNumOfCpus() + +   
> >> def getMaxNumOfThreads(self, command, params): +        """ +       
> >> Get the max number of threads that the server can tolerate +        """
> >> +        return bb.helper.CpuInfo.getNumOfCpus() *
> >> bb.helper.CpuInfo.coefficient
> >> 
> > 
> > I can understand needing to query the number of cpus but this last
> > function seems rather arbitrary. If you want to do bounds checking, I'd
> > suggest just adding the factor of 4 into the UI.
> OK
> 
> > I'm still not convinced
> > we should be setting any value for this though, or doing any bounds
> > checking on the value.
> Which value? We hope when the UI starts up, the UI can take advantage of cpus for build and set it as default.
> But users can change between 0 ~ max num of threads.

You can either:

a) Make it a text input (and then attempt to convert to an int, error if
its not an int). This means you don't need a maximum value.
b) In the UI, just show options up to 4 * cpu count.

I'd probably prefer a) but I can see why b) is easier. I don't see any
value in encoding "4" in the bitbake server though.

Cheers,

Richard

Patch

diff --git a/bitbake/lib/bb/helper.py b/bitbake/lib/bb/helper.py
new file mode 100644
index 0000000..291158b
--- /dev/null
+++ b/bitbake/lib/bb/helper.py
@@ -0,0 +1,39 @@ 
+#
+# Helper for BitBake Graphical GTK User Interface
+#
+# Copyright (C) 2011        Intel Corporation
+#
+# Authored by Shane Wang <shane.wang@intel.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+
+class CpuInfo(object):
+
+    coefficient = 4
+
+    @classmethod
+    def getNumOfCpus(cls):
+        pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores')
+        num = len(pfile.readlines())
+        return num
+
+    @classmethod
+    def getNumOfCpuCores(cls):
+        pfile = os.popen('cat /proc/cpuinfo | grep cpu\ cores | cut -d: -f2')
+        contents = pfile.readlines()
+        num = int(contents[0])
+        return num
+
diff --git a/lib/bb/command.py b/lib/bb/command.py
index 05555c5..eaf8236 100644
--- a/lib/bb/command.py
+++ b/lib/bb/command.py
@@ -30,6 +30,7 @@  Commands are queued in a CommandQueue
 
 import bb.event
 import bb.cooker
+import bb.helper
 
 class CommandCompleted(bb.event.Event):
     pass
@@ -173,6 +174,17 @@  class CommandsSync:
         """
         command.cooker.reset()
 
+    def getDefaultNumOfThreads(self, command, params):
+        """
+        Get the default number of threads on the server = number of CPUs
+        """
+        return bb.helper.CpuInfo.getNumOfCpus()
+
+    def getMaxNumOfThreads(self, command, params):
+        """
+        Get the max number of threads that the server can tolerate
+        """
+        return bb.helper.CpuInfo.getNumOfCpus() * bb.helper.CpuInfo.coefficient
 
 class CommandsAsync:
     """