Patchwork PR_Service: Improvement of persistance of PR service

login
register
mail settings
Submitter Dimitrios Katsaros
Date July 30, 2014, 10:27 p.m.
Message ID <1406759278-3258-1-git-send-email-dmtkatsa@gmail.com>
Download mbox | patch
Permalink /patch/76973/
State New
Headers show

Comments

Dimitrios Katsaros - July 30, 2014, 10:27 p.m.
The current implementation only commits database changes
after database exceptions or daemon shutdown. This patch adds
a delayed commit after every request finish, defaulting to 20 seconds.
I chose to make the sync on the consumer thread rather than the
producer. This is to avoid simultaneous access to the database.
I chose to use a separate timeout than the timeout defined for handle_request
in work_forever. This is because the consumer thread would reach the
handle_request before the quit flag was set. Thus for long timeouts
the stop_daemon function would have to wait for an equally long
amount of time.

This solution is based on this discussion :
https://www.mail-archive.com/openembedded-core@lists.openembedded.org/msg52932.html

Signed-off-by: Dimitrios Katsaros <dmtkatsa@gmail.com>
---
 bitbake/lib/prserv/serv.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)
Dimitrios Katsaros - Aug. 5, 2014, 3:11 p.m.
I just realised that there is a different mailing list and repo for
bitbake, I will be posting a new post with the corrected patch.


On Thu, Jul 31, 2014 at 12:27 AM, Dimitrios Katsaros <dmtkatsa@gmail.com>
wrote:

> The current implementation only commits database changes
> after database exceptions or daemon shutdown. This patch adds
> a delayed commit after every request finish, defaulting to 20 seconds.
> I chose to make the sync on the consumer thread rather than the
> producer. This is to avoid simultaneous access to the database.
> I chose to use a separate timeout than the timeout defined for
> handle_request
> in work_forever. This is because the consumer thread would reach the
> handle_request before the quit flag was set. Thus for long timeouts
> the stop_daemon function would have to wait for an equally long
> amount of time.
>
> This solution is based on this discussion :
>
> https://www.mail-archive.com/openembedded-core@lists.openembedded.org/msg52932.html
>
> Signed-off-by: Dimitrios Katsaros <dmtkatsa@gmail.com>
> ---
>  bitbake/lib/prserv/serv.py | 22 ++++++++++++++++++----
>  1 file changed, 18 insertions(+), 4 deletions(-)
>
> diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
> index 1e170ce..9c37339 100644
> --- a/bitbake/lib/prserv/serv.py
> +++ b/bitbake/lib/prserv/serv.py
> @@ -1,4 +1,4 @@
> -import os,sys,logging
> +import os,sys,logging, time
>  import signal, time, atexit, threading
>  from SimpleXMLRPCServer import SimpleXMLRPCServer,
> SimpleXMLRPCRequestHandler
>  import xmlrpclib
> @@ -36,7 +36,7 @@ singleton = None
>
>
>  class PRServer(SimpleXMLRPCServer):
> -    def __init__(self, dbfile, logfile, interface, daemon=True):
> +    def __init__(self, dbfile, logfile, interface, daemon=True,
> sync_timeout=20):
>          ''' constructor '''
>          SimpleXMLRPCServer.__init__(self, interface,
>                                      logRequests=False, allow_none=True)
> @@ -46,6 +46,8 @@ class PRServer(SimpleXMLRPCServer):
>          self.working_thread=None
>          self.host, self.port = self.socket.getsockname()
>          self.pidfile=PIDPREFIX % (self.host, self.port)
> +        self.dirty = False
> +        self.sync_timeout = sync_timeout
>
>          self.register_function(self.getPR, "getPR")
>          self.register_function(self.quit, "quit")
> @@ -70,8 +72,13 @@ class PRServer(SimpleXMLRPCServer):
>          while True:
>              (request, client_address) = self.requestqueue.get()
>              try:
> -                self.finish_request(request, client_address)
> -                self.shutdown_request(request)
> +                if request == "Sync":
> +                    self.table.sync()
> +                else:
> +                    self.finish_request(request, client_address)
> +                    self.shutdown_request(request)
> +                    self.time_from_dirty = time.time()
> +                    self.dirty = True
>              except:
>                  self.handle_error(request, client_address)
>                  self.shutdown_request(request)
> @@ -126,6 +133,13 @@ class PRServer(SimpleXMLRPCServer):
>          self.server_close()
>          return
>
> +    def handle_timeout(self):
> +        if self.dirty:
> +            # making the consumer thead perform sync to ensure thread
> safty
> +            if time.time() - self.time_from_dirty > self.sync_timeout:
> +                self.requestqueue.put(("Sync", None))
> +                self.dirty = False
> +
>      def start(self):
>          pid = self.daemonize()
>          # Ensure both the parent sees this and the child from the
> work_forever log entry above
> --
> 1.9.1
>
>

Patch

diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index 1e170ce..9c37339 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -1,4 +1,4 @@ 
-import os,sys,logging
+import os,sys,logging, time
 import signal, time, atexit, threading
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 import xmlrpclib
@@ -36,7 +36,7 @@  singleton = None
 
 
 class PRServer(SimpleXMLRPCServer):
-    def __init__(self, dbfile, logfile, interface, daemon=True):
+    def __init__(self, dbfile, logfile, interface, daemon=True, sync_timeout=20):
         ''' constructor '''
         SimpleXMLRPCServer.__init__(self, interface,
                                     logRequests=False, allow_none=True)
@@ -46,6 +46,8 @@  class PRServer(SimpleXMLRPCServer):
         self.working_thread=None
         self.host, self.port = self.socket.getsockname()
         self.pidfile=PIDPREFIX % (self.host, self.port)
+        self.dirty = False
+        self.sync_timeout = sync_timeout
 
         self.register_function(self.getPR, "getPR")
         self.register_function(self.quit, "quit")
@@ -70,8 +72,13 @@  class PRServer(SimpleXMLRPCServer):
         while True:
             (request, client_address) = self.requestqueue.get()
             try:
-                self.finish_request(request, client_address)
-                self.shutdown_request(request)
+                if request == "Sync":
+                    self.table.sync()
+                else:
+                    self.finish_request(request, client_address)
+                    self.shutdown_request(request)
+                    self.time_from_dirty = time.time()
+                    self.dirty = True
             except:
                 self.handle_error(request, client_address)
                 self.shutdown_request(request)
@@ -126,6 +133,13 @@  class PRServer(SimpleXMLRPCServer):
         self.server_close()
         return
 
+    def handle_timeout(self):
+        if self.dirty:
+            # making the consumer thead perform sync to ensure thread safty
+            if time.time() - self.time_from_dirty > self.sync_timeout:
+                self.requestqueue.put(("Sync", None))
+                self.dirty = False
+
     def start(self):
         pid = self.daemonize()
         # Ensure both the parent sees this and the child from the work_forever log entry above