[bitbake-devel,10/10] server/process: Add bitbake-server and exec() a new server process

Submitted by Richard Purdie on Aug. 24, 2020, 4:53 p.m. | Patch ID: 175622

Details

Message ID 20200824165331.1373725-10-richard.purdie@linuxfoundation.org
State New
Headers show

Commit Message

Richard Purdie Aug. 24, 2020, 4:53 p.m.
Trying to have a new python process forked off an original doesn't work
out well and ends up having race issues. To avoid this, exec() a new
bitbake server process. This starts with a fresh python interpreter
and resolves various atexit and other multiprocessing issues once
and for all.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/server/process.py | 92 +++++++++++++++++++++++-----------------
 1 file changed, 53 insertions(+), 39 deletions(-)

Patch hide | download patch | download mbox

diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 915651084e..d248570cd2 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -38,7 +38,7 @@  class ProcessServer():
     profile_filename = "profile.log"
     profile_processed_filename = "profile.log.processed"
 
-    def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
+    def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
         self.command_channel = False
         self.command_channel_reply = False
         self.quit = False
@@ -54,6 +54,7 @@  class ProcessServer():
         self._idlefuns = {}
 
         self.bitbake_lock = lock
+        self.bitbake_lock_name = lockname
         self.sock = sock
         self.sockname = sockname
 
@@ -259,7 +260,7 @@  class ProcessServer():
 
         # Finally release the lockfile but warn about other processes holding it open
         lock = self.bitbake_lock
-        lockfile = lock.name
+        lockfile = self.bitbake_lock_name
         lock.close()
         lock = None
 
@@ -393,9 +394,10 @@  class BitBakeProcessServerConnection(object):
         self.connection.recv.close()
         return
 
+start_log_format = '--- Starting bitbake server pid %s at %s ---'
+start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
+
 class BitBakeServer(object):
-    start_log_format = '--- Starting bitbake server pid %s at %s ---'
-    start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
 
     def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
 
@@ -429,7 +431,7 @@  class BitBakeServer(object):
             ready.close()
             bb.error("Unable to start bitbake server (%s)" % str(r))
             if os.path.exists(logfile):
-                logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
+                logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
                 started = False
                 lines = []
                 lastlines = []
@@ -441,7 +443,7 @@  class BitBakeServer(object):
                             lastlines.append(line)
                             res = logstart_re.match(line.rstrip())
                             if res:
-                                ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
+                                ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
                                 if ldatetime >= startdatetime:
                                     started = True
                                     lines.append(line)
@@ -462,42 +464,54 @@  class BitBakeServer(object):
         ready.close()
 
     def _startServer(self):
-        print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
-        sys.stdout.flush()
+        os.close(self.readypipe)
+        os.set_inheritable(self.bitbake_lock.fileno(), True)
+        os.set_inheritable(self.readypipein, True)
+        os.execlp("bitbake-server", "bitbake-server", "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
-        try:
-            # Create server control socket
-            if os.path.exists(self.sockname):
-                os.unlink(self.sockname)
+def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
 
-            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            # AF_UNIX has path length issues so chdir here to workaround
-            cwd = os.getcwd()
-            try:
-                os.chdir(os.path.dirname(self.sockname))
-                sock.bind(os.path.basename(self.sockname))
-            finally:
-                os.chdir(cwd)
-            sock.listen(1)
-
-            server = ProcessServer(self.bitbake_lock, sock, self.sockname, self.server_timeout, self.xmlrpcinterface)
-            os.close(self.readypipe)
-            writer = ConnectionWriter(self.readypipein)
-            try:
-                self.cooker = bb.cooker.BBCooker(self.featureset, server.register_idle_function)
-            except bb.BBHandledException:
-                return None
-            writer.send("r")
-            writer.close()
-            server.cooker = self.cooker
-            print("Started bitbake server pid %d" % os.getpid())
-            sys.stdout.flush()
-
-            server.run()
+    import bb.cookerdata
+    import bb.cooker
+
+    print(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format)))
+    sys.stdout.flush()
+
+    try:
+        bitbake_lock = os.fdopen(lockfd, "w")
+
+        # Create server control socket
+        if os.path.exists(sockname):
+            os.unlink(sockname)
+
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        # AF_UNIX has path length issues so chdir here to workaround
+        cwd = os.getcwd()
+        try:
+            os.chdir(os.path.dirname(sockname))
+            sock.bind(os.path.basename(sockname))
         finally:
-            # Flush any ,essages/errors to the logfile before exit
-            sys.stdout.flush()
-            sys.stderr.flush()
+            os.chdir(cwd)
+        sock.listen(1)
+
+        server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface)
+        writer = ConnectionWriter(readypipeinfd)
+        try:
+            featureset = []
+            cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
+        except bb.BBHandledException:
+            return None
+        writer.send("r")
+        writer.close()
+        server.cooker = cooker
+        print("Started bitbake server pid %d" % os.getpid())
+        sys.stdout.flush()
+
+        server.run()
+    finally:
+        # Flush any ,essages/errors to the logfile before exit
+        sys.stdout.flush()
+        sys.stderr.flush()
 
 def connectProcessServer(sockname, featureset):
     # Connect to socket

Comments

Joshua Watt Aug. 25, 2020, 4:06 p.m.
On Mon, Aug 24, 2020 at 11:53 AM Richard Purdie
<richard.purdie@linuxfoundation.org> wrote:
>
> Trying to have a new python process forked off an original doesn't work
> out well and ends up having race issues. To avoid this, exec() a new
> bitbake server process. This starts with a fresh python interpreter
> and resolves various atexit and other multiprocessing issues once
> and for all.
>
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  lib/bb/server/process.py | 92 +++++++++++++++++++++++-----------------
>  1 file changed, 53 insertions(+), 39 deletions(-)
>
> diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
> index 915651084e..d248570cd2 100644
> --- a/lib/bb/server/process.py
> +++ b/lib/bb/server/process.py
> @@ -38,7 +38,7 @@ class ProcessServer():
>      profile_filename = "profile.log"
>      profile_processed_filename = "profile.log.processed"
>
> -    def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
> +    def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
>          self.command_channel = False
>          self.command_channel_reply = False
>          self.quit = False
> @@ -54,6 +54,7 @@ class ProcessServer():
>          self._idlefuns = {}
>
>          self.bitbake_lock = lock
> +        self.bitbake_lock_name = lockname
>          self.sock = sock
>          self.sockname = sockname
>
> @@ -259,7 +260,7 @@ class ProcessServer():
>
>          # Finally release the lockfile but warn about other processes holding it open
>          lock = self.bitbake_lock
> -        lockfile = lock.name
> +        lockfile = self.bitbake_lock_name
>          lock.close()
>          lock = None
>
> @@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object):
>          self.connection.recv.close()
>          return
>
> +start_log_format = '--- Starting bitbake server pid %s at %s ---'
> +start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
> +
>  class BitBakeServer(object):
> -    start_log_format = '--- Starting bitbake server pid %s at %s ---'
> -    start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
>
>      def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
>
> @@ -429,7 +431,7 @@ class BitBakeServer(object):
>              ready.close()
>              bb.error("Unable to start bitbake server (%s)" % str(r))
>              if os.path.exists(logfile):
> -                logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
> +                logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
>                  started = False
>                  lines = []
>                  lastlines = []
> @@ -441,7 +443,7 @@ class BitBakeServer(object):
>                              lastlines.append(line)
>                              res = logstart_re.match(line.rstrip())
>                              if res:
> -                                ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
> +                                ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
>                                  if ldatetime >= startdatetime:
>                                      started = True
>                                      lines.append(line)
> @@ -462,42 +464,54 @@ class BitBakeServer(object):
>          ready.close()
>
>      def _startServer(self):
> -        print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
> -        sys.stdout.flush()
> +        os.close(self.readypipe)
> +        os.set_inheritable(self.bitbake_lock.fileno(), True)
> +        os.set_inheritable(self.readypipein, True)
> +        os.execlp("bitbake-server", "bitbake-server", "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))

You may want to do:

 os.execl(sys.executable, sys.executable, "bitbake-server", ....)

This ensures that the same python interpreter is used for the server
and UI front end.

>
> -        try:
> -            # Create server control socket
> -            if os.path.exists(self.sockname):
> -                os.unlink(self.sockname)
> +def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
>
> -            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
> -            # AF_UNIX has path length issues so chdir here to workaround
> -            cwd = os.getcwd()
> -            try:
> -                os.chdir(os.path.dirname(self.sockname))
> -                sock.bind(os.path.basename(self.sockname))
> -            finally:
> -                os.chdir(cwd)
> -            sock.listen(1)
> -
> -            server = ProcessServer(self.bitbake_lock, sock, self.sockname, self.server_timeout, self.xmlrpcinterface)
> -            os.close(self.readypipe)
> -            writer = ConnectionWriter(self.readypipein)
> -            try:
> -                self.cooker = bb.cooker.BBCooker(self.featureset, server.register_idle_function)
> -            except bb.BBHandledException:
> -                return None
> -            writer.send("r")
> -            writer.close()
> -            server.cooker = self.cooker
> -            print("Started bitbake server pid %d" % os.getpid())
> -            sys.stdout.flush()
> -
> -            server.run()
> +    import bb.cookerdata
> +    import bb.cooker
> +
> +    print(start_log_format % (os.getpid(), datetime.datetime.now().strftime(start_log_datetime_format)))
> +    sys.stdout.flush()
> +
> +    try:
> +        bitbake_lock = os.fdopen(lockfd, "w")
> +
> +        # Create server control socket
> +        if os.path.exists(sockname):
> +            os.unlink(sockname)
> +
> +        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
> +        # AF_UNIX has path length issues so chdir here to workaround
> +        cwd = os.getcwd()
> +        try:
> +            os.chdir(os.path.dirname(sockname))
> +            sock.bind(os.path.basename(sockname))
>          finally:
> -            # Flush any ,essages/errors to the logfile before exit
> -            sys.stdout.flush()
> -            sys.stderr.flush()
> +            os.chdir(cwd)
> +        sock.listen(1)
> +
> +        server = ProcessServer(bitbake_lock, lockname, sock, sockname, server_timeout, xmlrpcinterface)
> +        writer = ConnectionWriter(readypipeinfd)
> +        try:
> +            featureset = []
> +            cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
> +        except bb.BBHandledException:
> +            return None
> +        writer.send("r")
> +        writer.close()
> +        server.cooker = cooker
> +        print("Started bitbake server pid %d" % os.getpid())
> +        sys.stdout.flush()
> +
> +        server.run()
> +    finally:
> +        # Flush any ,essages/errors to the logfile before exit
> +        sys.stdout.flush()
> +        sys.stderr.flush()
>
>  def connectProcessServer(sockname, featureset):
>      # Connect to socket
> --
> 2.25.1
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#11635): https://lists.openembedded.org/g/bitbake-devel/message/11635
Mute This Topic: https://lists.openembedded.org/mt/76389544/3617530
Group Owner: bitbake-devel+owner@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub  [oe-patchwork@oe-patch.openembedded.org]
-=-=-=-=-=-=-=-=-=-=-=-
Richard Purdie Aug. 25, 2020, 5:06 p.m.
On Tue, 2020-08-25 at 11:06 -0500, Joshua Watt wrote:
> On Mon, Aug 24, 2020 at 11:53 AM Richard Purdie
> <richard.purdie@linuxfoundation.org> wrote:
> > Trying to have a new python process forked off an original doesn't work
> > out well and ends up having race issues. To avoid this, exec() a new
> > bitbake server process. This starts with a fresh python interpreter
> > and resolves various atexit and other multiprocessing issues once
> > and for all.
> > 
> > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > ---
> >  lib/bb/server/process.py | 92 +++++++++++++++++++++++-----------------
> >  1 file changed, 53 insertions(+), 39 deletions(-)
> > 
> > diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
> > index 915651084e..d248570cd2 100644
> > --- a/lib/bb/server/process.py
> > +++ b/lib/bb/server/process.py
> > @@ -38,7 +38,7 @@ class ProcessServer():
> >      profile_filename = "profile.log"
> >      profile_processed_filename = "profile.log.processed"
> > 
> > -    def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
> > +    def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
> >          self.command_channel = False
> >          self.command_channel_reply = False
> >          self.quit = False
> > @@ -54,6 +54,7 @@ class ProcessServer():
> >          self._idlefuns = {}
> > 
> >          self.bitbake_lock = lock
> > +        self.bitbake_lock_name = lockname
> >          self.sock = sock
> >          self.sockname = sockname
> > 
> > @@ -259,7 +260,7 @@ class ProcessServer():
> > 
> >          # Finally release the lockfile but warn about other processes holding it open
> >          lock = self.bitbake_lock
> > -        lockfile = lock.name
> > +        lockfile = self.bitbake_lock_name
> >          lock.close()
> >          lock = None
> > 
> > @@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object):
> >          self.connection.recv.close()
> >          return
> > 
> > +start_log_format = '--- Starting bitbake server pid %s at %s ---'
> > +start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
> > +
> >  class BitBakeServer(object):
> > -    start_log_format = '--- Starting bitbake server pid %s at %s ---'
> > -    start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
> > 
> >      def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
> > 
> > @@ -429,7 +431,7 @@ class BitBakeServer(object):
> >              ready.close()
> >              bb.error("Unable to start bitbake server (%s)" % str(r))
> >              if os.path.exists(logfile):
> > -                logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
> > +                logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
> >                  started = False
> >                  lines = []
> >                  lastlines = []
> > @@ -441,7 +443,7 @@ class BitBakeServer(object):
> >                              lastlines.append(line)
> >                              res = logstart_re.match(line.rstrip())
> >                              if res:
> > -                                ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
> > +                                ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
> >                                  if ldatetime >= startdatetime:
> >                                      started = True
> >                                      lines.append(line)
> > @@ -462,42 +464,54 @@ class BitBakeServer(object):
> >          ready.close()
> > 
> >      def _startServer(self):
> > -        print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
> > -        sys.stdout.flush()
> > +        os.close(self.readypipe)
> > +        os.set_inheritable(self.bitbake_lock.fileno(), True)
> > +        os.set_inheritable(self.readypipein, True)
> > +        os.execlp("bitbake-server", "bitbake-server", "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
> 
> You may want to do:
> 
>  os.execl(sys.executable, sys.executable, "bitbake-server", ....)
> 
> This ensures that the same python interpreter is used for the server
> and UI front end.

Yes, good point, although leaving arg0 as bitbake-server is probably
helpful for debugging so you can see the server process in a process
listing?

I had to change the code slightly in master-next to use execl() with
"../../../bitbake-server" based upon __file__ as it would break in
cases where we've messed around with PATH.

Cheers,

Richard
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#11637): https://lists.openembedded.org/g/bitbake-devel/message/11637
Mute This Topic: https://lists.openembedded.org/mt/76389544/3617530
Group Owner: bitbake-devel+owner@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub  [oe-patchwork@oe-patch.openembedded.org]
-=-=-=-=-=-=-=-=-=-=-=-
Joshua Watt Aug. 25, 2020, 5:19 p.m.
On 8/25/20 12:06 PM, Richard Purdie wrote:
> On Tue, 2020-08-25 at 11:06 -0500, Joshua Watt wrote:
>> On Mon, Aug 24, 2020 at 11:53 AM Richard Purdie
>> <richard.purdie@linuxfoundation.org> wrote:
>>> Trying to have a new python process forked off an original doesn't work
>>> out well and ends up having race issues. To avoid this, exec() a new
>>> bitbake server process. This starts with a fresh python interpreter
>>> and resolves various atexit and other multiprocessing issues once
>>> and for all.
>>>
>>> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
>>> ---
>>>   lib/bb/server/process.py | 92 +++++++++++++++++++++++-----------------
>>>   1 file changed, 53 insertions(+), 39 deletions(-)
>>>
>>> diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
>>> index 915651084e..d248570cd2 100644
>>> --- a/lib/bb/server/process.py
>>> +++ b/lib/bb/server/process.py
>>> @@ -38,7 +38,7 @@ class ProcessServer():
>>>       profile_filename = "profile.log"
>>>       profile_processed_filename = "profile.log.processed"
>>>
>>> -    def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
>>> +    def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
>>>           self.command_channel = False
>>>           self.command_channel_reply = False
>>>           self.quit = False
>>> @@ -54,6 +54,7 @@ class ProcessServer():
>>>           self._idlefuns = {}
>>>
>>>           self.bitbake_lock = lock
>>> +        self.bitbake_lock_name = lockname
>>>           self.sock = sock
>>>           self.sockname = sockname
>>>
>>> @@ -259,7 +260,7 @@ class ProcessServer():
>>>
>>>           # Finally release the lockfile but warn about other processes holding it open
>>>           lock = self.bitbake_lock
>>> -        lockfile = lock.name
>>> +        lockfile = self.bitbake_lock_name
>>>           lock.close()
>>>           lock = None
>>>
>>> @@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object):
>>>           self.connection.recv.close()
>>>           return
>>>
>>> +start_log_format = '--- Starting bitbake server pid %s at %s ---'
>>> +start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
>>> +
>>>   class BitBakeServer(object):
>>> -    start_log_format = '--- Starting bitbake server pid %s at %s ---'
>>> -    start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
>>>
>>>       def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
>>>
>>> @@ -429,7 +431,7 @@ class BitBakeServer(object):
>>>               ready.close()
>>>               bb.error("Unable to start bitbake server (%s)" % str(r))
>>>               if os.path.exists(logfile):
>>> -                logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
>>> +                logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
>>>                   started = False
>>>                   lines = []
>>>                   lastlines = []
>>> @@ -441,7 +443,7 @@ class BitBakeServer(object):
>>>                               lastlines.append(line)
>>>                               res = logstart_re.match(line.rstrip())
>>>                               if res:
>>> -                                ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
>>> +                                ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
>>>                                   if ldatetime >= startdatetime:
>>>                                       started = True
>>>                                       lines.append(line)
>>> @@ -462,42 +464,54 @@ class BitBakeServer(object):
>>>           ready.close()
>>>
>>>       def _startServer(self):
>>> -        print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
>>> -        sys.stdout.flush()
>>> +        os.close(self.readypipe)
>>> +        os.set_inheritable(self.bitbake_lock.fileno(), True)
>>> +        os.set_inheritable(self.readypipein, True)
>>> +        os.execlp("bitbake-server", "bitbake-server", "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
>> You may want to do:
>>
>>   os.execl(sys.executable, sys.executable, "bitbake-server", ....)
>>
>> This ensures that the same python interpreter is used for the server
>> and UI front end.
> Yes, good point, although leaving arg0 as bitbake-server is probably
> helpful for debugging so you can see the server process in a process
> listing?

Ya, that's seems fine:

  os.execl(sys.executable, "bitbake-server", bitbake_server_path, ....)

looks like it would work

>
> I had to change the code slightly in master-next to use execl() with
> "../../../bitbake-server" based upon __file__ as it would break in
> cases where we've messed around with PATH.
>
> Cheers,
>
> Richard
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#11638): https://lists.openembedded.org/g/bitbake-devel/message/11638
Mute This Topic: https://lists.openembedded.org/mt/76389544/3617530
Group Owner: bitbake-devel+owner@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub  [oe-patchwork@oe-patch.openembedded.org]
-=-=-=-=-=-=-=-=-=-=-=-
Richard Purdie Aug. 25, 2020, 5:23 p.m.
On Tue, 2020-08-25 at 12:19 -0500, Joshua Watt wrote:
> On 8/25/20 12:06 PM, Richard Purdie wrote:
> > On Tue, 2020-08-25 at 11:06 -0500, Joshua Watt wrote:
> > > On Mon, Aug 24, 2020 at 11:53 AM Richard Purdie
> > > <richard.purdie@linuxfoundation.org> wrote:
> > > > Trying to have a new python process forked off an original doesn't work
> > > > out well and ends up having race issues. To avoid this, exec() a new
> > > > bitbake server process. This starts with a fresh python interpreter
> > > > and resolves various atexit and other multiprocessing issues once
> > > > and for all.
> > > > 
> > > > Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> > > > ---
> > > >   lib/bb/server/process.py | 92 +++++++++++++++++++++++-----------------
> > > >   1 file changed, 53 insertions(+), 39 deletions(-)
> > > > 
> > > > diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
> > > > index 915651084e..d248570cd2 100644
> > > > --- a/lib/bb/server/process.py
> > > > +++ b/lib/bb/server/process.py
> > > > @@ -38,7 +38,7 @@ class ProcessServer():
> > > >       profile_filename = "profile.log"
> > > >       profile_processed_filename = "profile.log.processed"
> > > > 
> > > > -    def __init__(self, lock, sock, sockname, server_timeout, xmlrpcinterface):
> > > > +    def __init__(self, lock, lockname, sock, sockname, server_timeout, xmlrpcinterface):
> > > >           self.command_channel = False
> > > >           self.command_channel_reply = False
> > > >           self.quit = False
> > > > @@ -54,6 +54,7 @@ class ProcessServer():
> > > >           self._idlefuns = {}
> > > > 
> > > >           self.bitbake_lock = lock
> > > > +        self.bitbake_lock_name = lockname
> > > >           self.sock = sock
> > > >           self.sockname = sockname
> > > > 
> > > > @@ -259,7 +260,7 @@ class ProcessServer():
> > > > 
> > > >           # Finally release the lockfile but warn about other processes holding it open
> > > >           lock = self.bitbake_lock
> > > > -        lockfile = lock.name
> > > > +        lockfile = self.bitbake_lock_name
> > > >           lock.close()
> > > >           lock = None
> > > > 
> > > > @@ -393,9 +394,10 @@ class BitBakeProcessServerConnection(object):
> > > >           self.connection.recv.close()
> > > >           return
> > > > 
> > > > +start_log_format = '--- Starting bitbake server pid %s at %s ---'
> > > > +start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
> > > > +
> > > >   class BitBakeServer(object):
> > > > -    start_log_format = '--- Starting bitbake server pid %s at %s ---'
> > > > -    start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
> > > > 
> > > >       def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
> > > > 
> > > > @@ -429,7 +431,7 @@ class BitBakeServer(object):
> > > >               ready.close()
> > > >               bb.error("Unable to start bitbake server (%s)" % str(r))
> > > >               if os.path.exists(logfile):
> > > > -                logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
> > > > +                logstart_re = re.compile(start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
> > > >                   started = False
> > > >                   lines = []
> > > >                   lastlines = []
> > > > @@ -441,7 +443,7 @@ class BitBakeServer(object):
> > > >                               lastlines.append(line)
> > > >                               res = logstart_re.match(line.rstrip())
> > > >                               if res:
> > > > -                                ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
> > > > +                                ldatetime = datetime.datetime.strptime(res.group(2), start_log_datetime_format)
> > > >                                   if ldatetime >= startdatetime:
> > > >                                       started = True
> > > >                                       lines.append(line)
> > > > @@ -462,42 +464,54 @@ class BitBakeServer(object):
> > > >           ready.close()
> > > > 
> > > >       def _startServer(self):
> > > > -        print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
> > > > -        sys.stdout.flush()
> > > > +        os.close(self.readypipe)
> > > > +        os.set_inheritable(self.bitbake_lock.fileno(), True)
> > > > +        os.set_inheritable(self.readypipein, True)
> > > > +        os.execlp("bitbake-server", "bitbake-server", "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
> > > You may want to do:
> > > 
> > >   os.execl(sys.executable, sys.executable, "bitbake-server", ....)
> > > 
> > > This ensures that the same python interpreter is used for the server
> > > and UI front end.
> > Yes, good point, although leaving arg0 as bitbake-server is probably
> > helpful for debugging so you can see the server process in a process
> > listing?
> 
> Ya, that's seems fine:
> 
>   os.execl(sys.executable, "bitbake-server", bitbake_server_path, ....)
> 
> looks like it would work

It does. I'm handling this as a followup patch since the original set
was tested and ready to merge (now merged).

This one will test and then merge if all the corner cases work out ok.

Cheers,

Richard
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#11640): https://lists.openembedded.org/g/bitbake-devel/message/11640
Mute This Topic: https://lists.openembedded.org/mt/76389544/3617530
Group Owner: bitbake-devel+owner@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub  [oe-patchwork@oe-patch.openembedded.org]
-=-=-=-=-=-=-=-=-=-=-=-