From patchwork Fri Apr 12 09:02:23 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42248 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 36992C4345F for ; Fri, 12 Apr 2024 09:02:52 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web11.42175.1712912570044782711 for ; Fri, 12 Apr 2024 02:02:50 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=JZTGiuZB; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 054BEFF808; Fri, 12 Apr 2024 09:02:47 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912568; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=D121oOMAnQx2s3DAmEE2+o9eIQCqENaK5db9wZMf7LU=; b=JZTGiuZBBrO2UG5ONM4BHYNDOVOQW4JZQCbc4bH9MiB/ImaKtyVQRx6XtTMloo6558F/Mu Y6HoHdORvz7NL8keI0pvnXLCwe/2yigdYFf/F3pkM0WE6Sbbtu9IJlf91GEu8yKXxbSwxq TFbmNB64xPdlJTy9jRMlLUu0Ra/TJqORWrTrc2fJb6aigu3fLyhmKvApVS29FrUmKK4Mp3 YA8ug4pIt4QhxPNkezmpaqi5dKPGUHQjcGAb69xpeew5eE6831sazNUE05fWDKumPc5Frt dCRu4M1hXGhRpNOWctsqCK3teWdWz/eaZc4sL2Smm2L1LaBOB1Ml1fwCebNf/Q== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 01/12] prserv: simplify the PRServerClient() interface Date: Fri, 12 Apr 2024 11:02:23 +0200 Message-Id: <20240412090234.4110915-2-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:02:52 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16076 From: Michael Opdenacker serv.py: simplify the PRServerClient() interface by passing the server object instead of multiple arguments, and then retrieving the data through this object. This replicates what is done for ServerClient() in hashserv/server.py Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/prserv/serv.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 5fc8863f70..28af636966 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -20,16 +20,16 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid" singleton = None class PRServerClient(bb.asyncrpc.AsyncServerConnection): - def __init__(self, socket, table, read_only): - super().__init__(socket, 'PRSERVICE', logger) + def __init__(self, socket, server): + super().__init__(socket, 'PRSERVICE', server.logger) + self.server = server + self.handlers.update({ 'get-pr': self.handle_get_pr, 'import-one': self.handle_import_one, 'export': self.handle_export, 'is-readonly': self.handle_is_readonly, }) - self.table = table - self.read_only = read_only def validate_proto_version(self): return (self.proto_version == (1, 0)) @@ -38,10 +38,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): try: return await super().dispatch_message(msg) except: - self.table.sync() + self.server.table.sync() raise else: - self.table.sync_if_dirty() + self.server.table.sync_if_dirty() async def handle_get_pr(self, request): version = request['version'] @@ -50,7 +50,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = None try: - value = self.table.getValue(version, pkgarch, checksum) + value = self.server.table.getValue(version, pkgarch, checksum) response = {'value': value} except prserv.NotFoundError: logger.error("can not find value for (%s, %s)",version, checksum) @@ -61,13 +61,13 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): async def handle_import_one(self, request): response = None - if not self.read_only: + if not self.server.read_only: version = request['version'] pkgarch = request['pkgarch'] checksum = request['checksum'] value = request['value'] - value = self.table.importone(version, pkgarch, checksum, value) + value = self.server.table.importone(version, pkgarch, checksum, value) if value is not None: response = {'value': value} @@ -80,7 +80,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): colinfo = request['colinfo'] try: - (metainfo, datainfo) = self.table.export(version, pkgarch, checksum, colinfo) + (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) except sqlite3.Error as exc: logger.error(str(exc)) metainfo = datainfo = None @@ -88,7 +88,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): return {'metainfo': metainfo, 'datainfo': datainfo} async def handle_is_readonly(self, request): - return {'readonly': self.read_only} + return {'readonly': self.server.read_only} class PRServer(bb.asyncrpc.AsyncServer): def __init__(self, dbfile, read_only=False): @@ -98,7 +98,7 @@ class PRServer(bb.asyncrpc.AsyncServer): self.read_only = read_only def accept_client(self, socket): - return PRServerClient(socket, self.table, self.read_only) + return PRServerClient(socket, self) def start(self): tasks = super().start() From patchwork Fri Apr 12 09:02:24 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42251 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 447D9C00A94 for ; Fri, 12 Apr 2024 09:02:52 +0000 (UTC) Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) by mx.groups.io with SMTP id smtpd.web10.42601.1712912570916057825 for ; Fri, 12 Apr 2024 02:02:51 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=jompMrpT; spf=pass (domain: bootlin.com, ip: 217.70.183.194, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id D50564000E; Fri, 12 Apr 2024 09:02:48 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912569; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=vzJAsLHO0PdQWtxIN3rBMCpjEgtaLqecdv74Kd2qSDc=; b=jompMrpTLc+HtVkMYKlZ21rGtZkNPm89fyJiTIRXb7iokKS/r9FjaaF2J5CaesKidRdU9X JlIPaimZ+IlgMJIdnTPCLsOR3MaEmskwzN7o4N29l3Hs+UWEMnJIQc3Dw7BE3l7tUTHL4r dceGr1NTGz+U+RT39DJLqwXjzp32xc5yogM9LcpxjPq+1Sle7YEg8teNML3Ar5dCSfppmN gAHe2Lek5Kn/UTChzLZNiUaWMoV/pxkvkpJxTMxhSTgU5J/3Srz9Xlirk4ulLODv9JHWhx MKvZ1iUI7TeUUQebYQghmzM1uyLWY9nrgnOPrLmL0KLIDN5OcXfkaeKUCoj8EA== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 02/12] prserv: use double quotes by default Date: Fri, 12 Apr 2024 11:02:24 +0200 Message-Id: <20240412090234.4110915-3-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:02:52 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16077 From: Michael Opdenacker To aligh with the hashserv code Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- bin/bitbake-prserv | 4 +-- lib/prserv/__init__.py | 4 +-- lib/prserv/client.py | 20 ++++++------- lib/prserv/db.py | 32 ++++++++++----------- lib/prserv/serv.py | 64 +++++++++++++++++++++--------------------- 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/bin/bitbake-prserv b/bin/bitbake-prserv index 5be42f3ce5..8c3808fb20 100755 --- a/bin/bitbake-prserv +++ b/bin/bitbake-prserv @@ -11,14 +11,14 @@ import optparse import warnings warnings.simplefilter("default") -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib")) import prserv import prserv.serv __version__="1.0.0" -PRHOST_DEFAULT='0.0.0.0' +PRHOST_DEFAULT="0.0.0.0" PRPORT_DEFAULT=8585 def main(): diff --git a/lib/prserv/__init__.py b/lib/prserv/__init__.py index 38ced818ad..5790a8db1e 100644 --- a/lib/prserv/__init__.py +++ b/lib/prserv/__init__.py @@ -12,8 +12,8 @@ import sys,logging def init_logger(logfile, loglevel): numeric_level = getattr(logging, loglevel.upper(), None) if not isinstance(numeric_level, int): - raise ValueError('Invalid log level: %s' % loglevel) - FORMAT = '%(asctime)-15s %(message)s' + raise ValueError("Invalid log level: %s" % loglevel) + FORMAT = "%(asctime)-15s %(message)s" logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT) class NotFoundError(Exception): diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 6b81356fac..7bc5188c53 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -11,40 +11,40 @@ logger = logging.getLogger("BitBake.PRserv") class PRAsyncClient(bb.asyncrpc.AsyncClient): def __init__(self): - super().__init__('PRSERVICE', '1.0', logger) + super().__init__("PRSERVICE", "1.0", logger) async def getPR(self, version, pkgarch, checksum): response = await self.invoke( - {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}} + {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} ) if response: - return response['value'] + return response["value"] async def importone(self, version, pkgarch, checksum, value): response = await self.invoke( - {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}} + {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}} ) if response: - return response['value'] + return response["value"] async def export(self, version, pkgarch, checksum, colinfo): response = await self.invoke( - {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}} + {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo}} ) if response: - return (response['metainfo'], response['datainfo']) + return (response["metainfo"], response["datainfo"]) async def is_readonly(self): response = await self.invoke( - {'is-readonly': {}} + {"is-readonly": {}} ) if response: - return response['readonly'] + return response["readonly"] class PRClient(bb.asyncrpc.Client): def __init__(self): super().__init__() - self._add_methods('getPR', 'importone', 'export', 'is_readonly') + self._add_methods("getPR", "importone", "export", "is_readonly") def _get_async_client(self): return PRAsyncClient() diff --git a/lib/prserv/db.py b/lib/prserv/db.py index b4bda7078c..0859cf4f2c 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -64,7 +64,7 @@ class PRTable(object): try: return self.conn.execute(*query) except sqlite3.OperationalError as exc: - if 'is locked' in str(exc) and end > time.time(): + if "is locked" in str(exc) and end > time.time(): continue raise exc @@ -220,18 +220,18 @@ class PRTable(object): metainfo = {} #column info if colinfo: - metainfo['tbl_name'] = self.table - metainfo['core_ver'] = prserv.__version__ - metainfo['col_info'] = [] + metainfo["tbl_name"] = self.table + metainfo["core_ver"] = prserv.__version__ + metainfo["col_info"] = [] data = self._execute("PRAGMA table_info(%s);" % self.table) for row in data: col = {} - col['name'] = row['name'] - col['type'] = row['type'] - col['notnull'] = row['notnull'] - col['dflt_value'] = row['dflt_value'] - col['pk'] = row['pk'] - metainfo['col_info'].append(col) + col["name"] = row["name"] + col["type"] = row["type"] + col["notnull"] = row["notnull"] + col["dflt_value"] = row["dflt_value"] + col["pk"] = row["pk"] + metainfo["col_info"].append(col) #data info datainfo = [] @@ -261,12 +261,12 @@ class PRTable(object): else: data = self._execute(sqlstmt) for row in data: - if row['version']: + if row["version"]: col = {} - col['version'] = row['version'] - col['pkgarch'] = row['pkgarch'] - col['checksum'] = row['checksum'] - col['value'] = row['value'] + col["version"] = row["version"] + col["pkgarch"] = row["pkgarch"] + col["checksum"] = row["checksum"] + col["value"] = row["value"] datainfo.append(col) return (metainfo, datainfo) @@ -275,7 +275,7 @@ class PRTable(object): for line in self.conn.iterdump(): writeCount = writeCount + len(line) + 1 fd.write(line) - fd.write('\n') + fd.write("\n") return writeCount class PRData(object): diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 28af636966..49f5ae36db 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -21,14 +21,14 @@ singleton = None class PRServerClient(bb.asyncrpc.AsyncServerConnection): def __init__(self, socket, server): - super().__init__(socket, 'PRSERVICE', server.logger) + super().__init__(socket, "PRSERVICE", server.logger) self.server = server self.handlers.update({ - 'get-pr': self.handle_get_pr, - 'import-one': self.handle_import_one, - 'export': self.handle_export, - 'is-readonly': self.handle_is_readonly, + "get-pr": self.handle_get_pr, + "import-one": self.handle_import_one, + "export": self.handle_export, + "is-readonly": self.handle_is_readonly, }) def validate_proto_version(self): @@ -44,14 +44,14 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): self.server.table.sync_if_dirty() async def handle_get_pr(self, request): - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] response = None try: value = self.server.table.getValue(version, pkgarch, checksum) - response = {'value': value} + response = {"value": value} except prserv.NotFoundError: logger.error("can not find value for (%s, %s)",version, checksum) except sqlite3.Error as exc: @@ -62,22 +62,22 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): async def handle_import_one(self, request): response = None if not self.server.read_only: - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] - value = request['value'] + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] + value = request["value"] value = self.server.table.importone(version, pkgarch, checksum, value) if value is not None: - response = {'value': value} + response = {"value": value} return response async def handle_export(self, request): - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] - colinfo = request['colinfo'] + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] + colinfo = request["colinfo"] try: (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) @@ -85,10 +85,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): logger.error(str(exc)) metainfo = datainfo = None - return {'metainfo': metainfo, 'datainfo': datainfo} + return {"metainfo": metainfo, "datainfo": datainfo} async def handle_is_readonly(self, request): - return {'readonly': self.server.read_only} + return {"readonly": self.server.read_only} class PRServer(bb.asyncrpc.AsyncServer): def __init__(self, dbfile, read_only=False): @@ -135,7 +135,7 @@ class PRServSingleton(object): if not self.prserv.address: raise PRServiceConfigError if not self.port: - self.port = int(self.prserv.address.rsplit(':', 1)[1]) + self.port = int(self.prserv.address.rsplit(":", 1)[1]) def run_as_daemon(func, pidfile, logfile): """ @@ -171,12 +171,12 @@ def run_as_daemon(func, pidfile, logfile): # stdout/stderr or it could be 'real' unix fd forking where we need # to physically close the fds to prevent the program launching us from # potentially hanging on a pipe. Handle both cases. - si = open('/dev/null', 'r') + si = open("/dev/null", "r") try: os.dup2(si.fileno(),sys.stdin.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stdin = si - so = open(logfile, 'a+') + so = open(logfile, "a+") try: os.dup2(so.fileno(),sys.stdout.fileno()) except (AttributeError, io.UnsupportedOperation): @@ -200,7 +200,7 @@ def run_as_daemon(func, pidfile, logfile): # write pidfile pid = str(os.getpid()) - with open(pidfile, 'w') as pf: + with open(pidfile, "w") as pf: pf.write("%s\n" % pid) func() @@ -245,12 +245,12 @@ def stop_daemon(host, port): # so at least advise the user which ports the corresponding server is listening ports = [] portstr = "" - for pf in glob.glob(PIDPREFIX % (ip,'*')): + for pf in glob.glob(PIDPREFIX % (ip, "*")): bn = os.path.basename(pf) root, _ = os.path.splitext(bn) - ports.append(root.split('_')[-1]) + ports.append(root.split("_")[-1]) if len(ports): - portstr = "Wrong port? Other ports listening at %s: %s" % (host, ' '.join(ports)) + portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports)) sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n" % (pidfile,portstr)) @@ -284,7 +284,7 @@ def is_running(pid): return True def is_local_special(host, port): - if (host == 'localhost' or host == '127.0.0.1') and not port: + if (host == "localhost" or host == "127.0.0.1") and not port: return True else: return False @@ -295,7 +295,7 @@ class PRServiceConfigError(Exception): def auto_start(d): global singleton - host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':'))) + host_params = list(filter(None, (d.getVar("PRSERV_HOST") or "").split(":"))) if not host_params: # Shutdown any existing PR Server auto_shutdown() @@ -304,7 +304,7 @@ def auto_start(d): if len(host_params) != 2: # Shutdown any existing PR Server auto_shutdown() - logger.critical('\n'.join(['PRSERV_HOST: incorrect format', + logger.critical("\n".join(["PRSERV_HOST: incorrect format", 'Usage: PRSERV_HOST = ":"'])) raise PRServiceConfigError @@ -357,8 +357,8 @@ def connect(host, port): global singleton - if host.strip().lower() == 'localhost' and not port: - host = 'localhost' + if host.strip().lower() == "localhost" and not port: + host = "localhost" port = singleton.port conn = client.PRClient() From patchwork Fri Apr 12 09:02:25 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42249 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 61465C04FFE for ; Fri, 12 Apr 2024 09:02:52 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web10.42603.1712912571713396234 for ; Fri, 12 Apr 2024 02:02:52 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=P4VeGRVp; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id ADCBDC0006; Fri, 12 Apr 2024 09:02:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912569; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=6znkIhcGuYGdGlFsT9PyihyxZjDmg4apUHRFvg+cl9c=; b=P4VeGRVpG65aYGHRUu1ysEtBIwAyoHfh5TP2flvZlxAktWM/niuncRneWgYmklxxDN7Su7 HFNIGvuzYh4pbYJD4JO5QG/SS53LyxOsHnA7mKzxeKWs1QbM13+NBAbfy9WrJ0j3xW1y9i +iKLH4APA3KERbCVKr1r/jzMQNQFNvSUFjqlMOP1IpGr2ZlkqqhA/2bOEREF2AC6rLZ+h7 u7Q35G8ZoM7ugbQPE9bhJEMFj3gzYkTyQJU5RXMtfg0c7lIpEXetlmowDXSUDlWnDoWv7d +aO1vtPvliD88r75cJb0BAfawyEreWHsiz4GGSsimetRSxCWSjibH9XZzn0ZOw== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 03/12] bitbake-prserv: replace deprecated optparse by argparse Date: Fri, 12 Apr 2024 11:02:25 +0200 Message-Id: <20240412090234.4110915-4-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:02:52 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16079 From: Michael Opdenacker optparse is deprecated since Python 2.7 Note that this is neither supposed to change the options supported by bitbake-prserv nor the way they are interpreted. Note that in the "--help" output, long options are now reported for example as "--host HOST" instead of "--host=HOST" but both are equivalent anyway, as they already were with optparse. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- bin/bitbake-prserv | 82 +++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/bin/bitbake-prserv b/bin/bitbake-prserv index 8c3808fb20..ad0a069401 100755 --- a/bin/bitbake-prserv +++ b/bin/bitbake-prserv @@ -7,7 +7,7 @@ import os import sys,logging -import optparse +import argparse import warnings warnings.simplefilter("default") @@ -16,40 +16,68 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib import prserv import prserv.serv -__version__="1.0.0" +VERSION = "1.1.0" PRHOST_DEFAULT="0.0.0.0" PRPORT_DEFAULT=8585 def main(): - parser = optparse.OptionParser( - version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__), - usage = "%prog < --start | --stop > [options]") + parser = argparse.ArgumentParser( + description="BitBake PR Server. Version=%s" % VERSION, + formatter_class=argparse.RawTextHelpFormatter) - parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store", - dest="dbfile", type="string", default="prserv.sqlite3") - parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store", - dest="logfile", type="string", default="prserv.log") - parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", - action = "store", type="string", dest="loglevel", default = "INFO") - parser.add_option("--start", help="start daemon", - action="store_true", dest="start") - parser.add_option("--stop", help="stop daemon", - action="store_true", dest="stop") - parser.add_option("--host", help="ip address to bind", action="store", - dest="host", type="string", default=PRHOST_DEFAULT) - parser.add_option("--port", help="port number(default: 8585)", action="store", - dest="port", type="int", default=PRPORT_DEFAULT) - parser.add_option("-r", "--read-only", help="open database in read-only mode", - action="store_true") + parser.add_argument( + "-f", + "--file", + default="prserv.sqlite3", + help="database filename (default: prserv.sqlite3)", + ) + parser.add_argument( + "-l", + "--log", + default="prserv.log", + help="log filename(default: prserv.log)", + ) + parser.add_argument( + "--loglevel", + default="INFO", + help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", + ) + parser.add_argument( + "--start", + action="store_true", + help="start daemon", + ) + parser.add_argument( + "--stop", + action="store_true", + help="stop daemon", + ) + parser.add_argument( + "--host", + help="ip address to bind", + default=PRHOST_DEFAULT, + ) + parser.add_argument( + "--port", + type=int, + default=PRPORT_DEFAULT, + help="port number (default: 8585)", + ) + parser.add_argument( + "-r", + "--read-only", + action="store_true", + help="open database in read-only mode", + ) - options, args = parser.parse_args(sys.argv) - prserv.init_logger(os.path.abspath(options.logfile),options.loglevel) + args = parser.parse_args() + prserv.init_logger(os.path.abspath(args.log), args.loglevel) - if options.start: - ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile), options.read_only) - elif options.stop: - ret=prserv.serv.stop_daemon(options.host, options.port) + if args.start: + ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only) + elif args.stop: + ret=prserv.serv.stop_daemon(args.host, args.port) else: ret=parser.print_help() return ret From patchwork Fri Apr 12 09:02:26 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42250 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 56FD0C41513 for ; Fri, 12 Apr 2024 09:02:52 +0000 (UTC) Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) by mx.groups.io with SMTP id smtpd.web11.42177.1712912571631269504 for ; Fri, 12 Apr 2024 02:02:51 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=bfqt4ZcS; spf=pass (domain: bootlin.com, ip: 217.70.183.194, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 27AFA40010; Fri, 12 Apr 2024 09:02:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912570; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=SIliTXomlk25yZ4h5haoq7Wu8fO4YIk4Iq8d8l0+8Ms=; b=bfqt4ZcSK2RJhrA4/z1XEEgbN3nAbxfBdpLbb3axkXzZu7jY+csZTnycaVbUw2+Sft3XV1 k0dwp5YBRfE6W2a+2GoEYK4kWTUwg+pFeyIj3yorKm2mGQiFp29Cv0htQZe6GVrCpUfS84 u/DeQWtelfz6G25dIGShk6Gzwoe8PUSLXu3dxVGZTKAb7aD1QcJwBjIdOajhtEn4w8FfrK deMCmvoeO/07OXeezXxv+0iZNKzir8ufebYqOuNdiax/YKhg1vPN6S0GNsxLX8VucLaR08 2LtJBfosIBjBRJuYkn3L97w2fukVserLxhEUpaKhDelDuQegoBYNKmn+7TH9SQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 04/12] prserv: use self.logger instead of logger directly Date: Fri, 12 Apr 2024 11:02:26 +0200 Message-Id: <20240412090234.4110915-5-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:02:52 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16078 From: Michael Opdenacker In both the PRServerClient and PRClient objects. This aligns with what is done in hashserv/server.py and makes it possible to benefit from possible specializations of the logger in the corresponding super classes, instead of using always the global logger. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/prserv/serv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 49f5ae36db..ab9a0a66df 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -53,9 +53,9 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): value = self.server.table.getValue(version, pkgarch, checksum) response = {"value": value} except prserv.NotFoundError: - logger.error("can not find value for (%s, %s)",version, checksum) + self.logger.error("can not find value for (%s, %s)",version, checksum) except sqlite3.Error as exc: - logger.error(str(exc)) + self.logger.error(str(exc)) return response @@ -82,7 +82,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): try: (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) except sqlite3.Error as exc: - logger.error(str(exc)) + self.logger.error(str(exc)) metainfo = datainfo = None return {"metainfo": metainfo, "datainfo": datainfo} @@ -105,7 +105,7 @@ class PRServer(bb.asyncrpc.AsyncServer): self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only) self.table = self.db["PRMAIN"] - logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % + self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % (self.dbfile, self.address, str(os.getpid()))) return tasks From patchwork Fri Apr 12 09:02:27 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42252 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 72E56C04FF6 for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web11.42178.1712912572056527402 for ; Fri, 12 Apr 2024 02:02:52 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=YknsdrKM; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 8A9C2C000E; Fri, 12 Apr 2024 09:02:50 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912570; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=sSWqVC9eYVk500ijX1Trr47OAQDUG1I8fUSpoqvkE5k=; b=YknsdrKMOc/stOzzT5EWfOdajcEkLgQ+PgCAKZRd8AbS2Bga53BqorBZeeAaMe1mqsj04Z 63A76GORoQnyTTAx8U0LQ8cAp081DX7iPciCLTiu7qEfHsjls/MVpTGAOQr2UiNhD6ZTpg rt6G3Up5+BsbYCmTS83wqMHHMsD7dXh/y560OBnqsmYy1T8kgtsfWdGXoDSAmj0vp1q9TG lpbhYBJvQ9mDvxaxtN+dJZyvw+r4yXOVqPShutawJ7fTXWSmFtl1dLrkyto8A3hif0w3MC iJwA1pov58cDJ9FFz1n+h+AOO2Sq9vAWj/50zKcWtS7jCE78blIQ1b4Ya6KiCQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 05/12] asyncrpc: include parse_address from hashserv Date: Fri, 12 Apr 2024 11:02:27 +0200 Message-Id: <20240412090234.4110915-6-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16080 From: Michael Opdenacker Moving the code and related definitions from hashserv/__init__.py to asyncrpc/client.py, allowing this function to be used in other asyncrpc clients. Signed-off-by: Michael Opdenacker Suggested-by: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/bb/asyncrpc/client.py | 23 +++++++++++++++++++++++ lib/hashserv/__init__.py | 27 +-------------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/lib/bb/asyncrpc/client.py b/lib/bb/asyncrpc/client.py index 29a5ab76aa..a350b4fb12 100644 --- a/lib/bb/asyncrpc/client.py +++ b/lib/bb/asyncrpc/client.py @@ -10,11 +10,34 @@ import json import os import socket import sys +import re import contextlib from threading import Thread from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK from .exceptions import ConnectionClosedError, InvokeError +UNIX_PREFIX = "unix://" +WS_PREFIX = "ws://" +WSS_PREFIX = "wss://" + +ADDR_TYPE_UNIX = 0 +ADDR_TYPE_TCP = 1 +ADDR_TYPE_WS = 2 + +def parse_address(addr): + if addr.startswith(UNIX_PREFIX): + return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],)) + elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX): + return (ADDR_TYPE_WS, (addr,)) + else: + m = re.match(r"\[(?P[^\]]*)\]:(?P\d+)$", addr) + if m is not None: + host = m.group("host") + port = m.group("port") + else: + host, port = addr.split(":") + + return (ADDR_TYPE_TCP, (host, int(port))) class AsyncClient(object): def __init__( diff --git a/lib/hashserv/__init__.py b/lib/hashserv/__init__.py index 552a33278f..74367eb6b4 100644 --- a/lib/hashserv/__init__.py +++ b/lib/hashserv/__init__.py @@ -5,39 +5,14 @@ import asyncio from contextlib import closing -import re import itertools import json from collections import namedtuple from urllib.parse import urlparse - -UNIX_PREFIX = "unix://" -WS_PREFIX = "ws://" -WSS_PREFIX = "wss://" - -ADDR_TYPE_UNIX = 0 -ADDR_TYPE_TCP = 1 -ADDR_TYPE_WS = 2 +from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS User = namedtuple("User", ("username", "permissions")) - -def parse_address(addr): - if addr.startswith(UNIX_PREFIX): - return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],)) - elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX): - return (ADDR_TYPE_WS, (addr,)) - else: - m = re.match(r"\[(?P[^\]]*)\]:(?P\d+)$", addr) - if m is not None: - host = m.group("host") - port = m.group("port") - else: - host, port = addr.split(":") - - return (ADDR_TYPE_TCP, (host, int(port))) - - def create_server( addr, dbname, From patchwork Fri Apr 12 09:02:28 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42257 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id AC63CC05024 for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web10.42605.1712912573436435911 for ; Fri, 12 Apr 2024 02:02:53 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=ZFj6q/Pv; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 7004E1BF20A; Fri, 12 Apr 2024 09:02:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912571; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=zN31MXJ6x0z63HKFCHdAEI3fsKCSq29RWTQJuXJc+HE=; b=ZFj6q/Pvj+I32F+2pCWrBqnHqEX288FRc3QsunxiNx/5VgoKDrdnRQo9IV14DTf0UIj6nW EcOTl1bHGSI7Ygprb4ICwbBoUXgJ+OVtyjwUgHsR+in9Qt2UwGs6CD0g8TfeZnElxsjgdI 7thVSlwBTAIQDM7oKBeLkKtu8iiHmccTCWhtJK3doItQAJ6JQMhA6V1/Nw73B3GQrPj9zG srTIlqhciK6pWPw83TkohbznLmqMT2eEA4ee3Xikbe4CzKJPabaFfU6fxAMt2jT37Gczjl Kz800yBZ2cmtIw0QYndvvxWtOLrrSwto2SDMlKIPIFwCpsCHrgBUVi5ucOZEIg== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 06/12] prserv: capitalization and spacing improvements Date: Fri, 12 Apr 2024 11:02:28 +0200 Message-Id: <20240412090234.4110915-7-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16082 From: Michael Opdenacker Choosing only one style of capitalization Add extra space after some commas too Remove idle spaces Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/prserv/__init__.py | 2 +- lib/prserv/db.py | 56 +++++++++++++++++++++--------------------- lib/prserv/serv.py | 10 ++++---- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/prserv/__init__.py b/lib/prserv/__init__.py index 5790a8db1e..0e0aa34d0e 100644 --- a/lib/prserv/__init__.py +++ b/lib/prserv/__init__.py @@ -7,7 +7,7 @@ __version__ = "1.0.0" import os, time -import sys,logging +import sys, logging def init_logger(logfile, loglevel): numeric_level = getattr(logging, loglevel.upper(), None) diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 0859cf4f2c..7bc2b2dc2d 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -38,9 +38,9 @@ class PRTable(object): self.read_only = read_only self.dirty = False if nohist: - self.table = "%s_nohist" % table + self.table = "%s_nohist" % table else: - self.table = "%s_hist" % table + self.table = "%s_hist" % table if self.read_only: table_exists = self._execute( @@ -78,7 +78,7 @@ class PRTable(object): self.sync() self.dirty = False - def _getValueHist(self, version, pkgarch, checksum): + def _get_value_hist(self, version, pkgarch, checksum): data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) row=data.fetchone() @@ -87,7 +87,7 @@ class PRTable(object): else: #no value found, try to insert if self.read_only: - data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), + data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) row = data.fetchone() if row is not None: @@ -96,9 +96,9 @@ class PRTable(object): return 0 try: - self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" - % (self.table,self.table), - (version,pkgarch, checksum,version, pkgarch)) + self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" + % (self.table, self.table), + (version, pkgarch, checksum, version, pkgarch)) except sqlite3.IntegrityError as exc: logger.error(str(exc)) @@ -112,10 +112,10 @@ class PRTable(object): else: raise prserv.NotFoundError - def _getValueNohist(self, version, pkgarch, checksum): + def _get_value_no_hist(self, version, pkgarch, checksum): data=self._execute("SELECT value FROM %s \ WHERE version=? AND pkgarch=? AND checksum=? AND \ - value >= (select max(value) from %s where version=? AND pkgarch=?);" + value >= (select max(value) from %s where version=? AND pkgarch=?);" % (self.table, self.table), (version, pkgarch, checksum, version, pkgarch)) row=data.fetchone() @@ -124,7 +124,7 @@ class PRTable(object): else: #no value found, try to insert if self.read_only: - data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), + data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) row = data.fetchone() if row is not None: @@ -133,8 +133,8 @@ class PRTable(object): return 0 try: - self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" - % (self.table,self.table), + self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" + % (self.table, self.table), (version, pkgarch, checksum, version, pkgarch)) except sqlite3.IntegrityError as exc: logger.error(str(exc)) @@ -150,17 +150,17 @@ class PRTable(object): else: raise prserv.NotFoundError - def getValue(self, version, pkgarch, checksum): + def get_value(self, version, pkgarch, checksum): if self.nohist: - return self._getValueNohist(version, pkgarch, checksum) + return self._get_value_no_hist(version, pkgarch, checksum) else: - return self._getValueHist(version, pkgarch, checksum) + return self._get_value_hist(version, pkgarch, checksum) - def _importHist(self, version, pkgarch, checksum, value): + def _import_hist(self, version, pkgarch, checksum, value): if self.read_only: return None - val = None + val = None data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) row = data.fetchone() @@ -183,27 +183,27 @@ class PRTable(object): val = row[0] return val - def _importNohist(self, version, pkgarch, checksum, value): + def _import_no_hist(self, version, pkgarch, checksum, value): if self.read_only: return None try: #try to insert self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), - (version, pkgarch, checksum,value)) + (version, pkgarch, checksum, value)) except sqlite3.IntegrityError as exc: #already have the record, try to update try: - self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value=?;" % self.table, - (version,pkgarch,checksum,value)) + (version, pkgarch, checksum, value)) row=data.fetchone() if row is not None: return row[0] @@ -212,13 +212,13 @@ class PRTable(object): def importone(self, version, pkgarch, checksum, value): if self.nohist: - return self._importNohist(version, pkgarch, checksum, value) + return self._import_no_hist(version, pkgarch, checksum, value) else: - return self._importHist(version, pkgarch, checksum, value) + return self._import_hist(version, pkgarch, checksum, value) def export(self, version, pkgarch, checksum, colinfo): metainfo = {} - #column info + #column info if colinfo: metainfo["tbl_name"] = self.table metainfo["core_ver"] = prserv.__version__ @@ -238,7 +238,7 @@ class PRTable(object): if self.nohist: sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ - (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \ + (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \ WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) else: sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table @@ -302,7 +302,7 @@ class PRData(object): def disconnect(self): self.connection.close() - def __getitem__(self,tblname): + def __getitem__(self, tblname): if not isinstance(tblname, str): raise TypeError("tblname argument must be a string, not '%s'" % type(tblname)) @@ -316,4 +316,4 @@ class PRData(object): if tblname in self._tables: del self._tables[tblname] logger.info("drop table %s" % (tblname)) - self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) + self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index ab9a0a66df..efb2e0cf93 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -50,7 +50,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = None try: - value = self.server.table.getValue(version, pkgarch, checksum) + value = self.server.table.get_value(version, pkgarch, checksum) response = {"value": value} except prserv.NotFoundError: self.logger.error("can not find value for (%s, %s)",version, checksum) @@ -173,16 +173,16 @@ def run_as_daemon(func, pidfile, logfile): # potentially hanging on a pipe. Handle both cases. si = open("/dev/null", "r") try: - os.dup2(si.fileno(),sys.stdin.fileno()) + os.dup2(si.fileno(), sys.stdin.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stdin = si so = open(logfile, "a+") try: - os.dup2(so.fileno(),sys.stdout.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stdout = so try: - os.dup2(so.fileno(),sys.stderr.fileno()) + os.dup2(so.fileno(), sys.stderr.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stderr = so @@ -253,7 +253,7 @@ def stop_daemon(host, port): portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports)) sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n" - % (pidfile,portstr)) + % (pidfile, portstr)) return 1 try: From patchwork Fri Apr 12 09:02:29 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42258 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 89A65C04FFE for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web10.42604.1712912573430602650 for ; Fri, 12 Apr 2024 02:02:53 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=BW+k1mj4; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id DFD0EFF806; Fri, 12 Apr 2024 09:02:51 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912572; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=m5k/BEDMv9ApMbsVgOA4DZUlF+i54NuQkfvdm/XCpGI=; b=BW+k1mj4uvqXPOr3fvVv4XZmzcHM95n4M0q8yhbYMmwnvxjc9zTfXc1Y0ISQbdD2oU1Z6A Ju4xt3T2Gz2tF4rwOunduwybc4VEn5jT3Kbwv61WuGaUpbsbu/mBWpluSP3efuT3aRETgu 5z45RsmKxeqPZQiReH6g3xb/JL8IQRaW5fzorhdo53YjpaK0l6FQlMsOZYh2cdUpk3kJI5 6e1x9bcPpQt///go0wdbq/Mu19UiRZv8CBuSekWtCJCl9CuahvIvqOJiOTOPqzRlxK9rIZ MyI6f/6vTbufykCfpFh3203TWfuTl570qMuVrnGKEWcpJ1iqf6j/EwfKWK2Ojg== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 07/12] prserv: add extra requests Date: Fri, 12 Apr 2024 11:02:29 +0200 Message-Id: <20240412090234.4110915-8-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16081 From: Michael Opdenacker Useful for connecting a PR server to an upstream one - "test-package" checks whether the specified package version and arch is known in the database. - "test-pr" checks a specified output hash is found in the database. Otherwise it returns 'None' instead of a new value. - "max-package-pr" returns the highest PR number for (version, arch) entries in the database, and None if not found Add new DB functions supporting the above, plus test_value() which tells whether a given value is available for the specified package and architecture. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni Reviewed-by: Bruce Ashfield --- lib/prserv/client.py | 23 +++++++++++++++++++++- lib/prserv/db.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ lib/prserv/serv.py | 28 +++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 7bc5188c53..8471ee3046 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -20,6 +20,27 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): if response: return response["value"] + async def test_pr(self, version, pkgarch, checksum): + response = await self.invoke( + {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} + ) + if response: + return response["value"] + + async def test_package(self, version, pkgarch): + response = await self.invoke( + {"test-package": {"version": version, "pkgarch": pkgarch}} + ) + if response: + return response["value"] + + async def max_package_pr(self, version, pkgarch): + response = await self.invoke( + {"max-package-pr": {"version": version, "pkgarch": pkgarch}} + ) + if response: + return response["value"] + async def importone(self, version, pkgarch, checksum, value): response = await self.invoke( {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}} @@ -44,7 +65,7 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): class PRClient(bb.asyncrpc.Client): def __init__(self): super().__init__() - self._add_methods("getPR", "importone", "export", "is_readonly") + self._add_methods("getPR", "test_pr", "test_package", "importone", "export", "is_readonly") def _get_async_client(self): return PRAsyncClient() diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 7bc2b2dc2d..7c200602ed 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -78,6 +78,52 @@ class PRTable(object): self.sync() self.dirty = False + def test_package(self, version, pkgarch): + """Returns whether the specified package version is found in the database for the specified architecture""" + + # Just returns the value if found or None otherwise + data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, + (version, pkgarch)) + row=data.fetchone() + if row is not None: + return True + else: + return False + + def test_value(self, version, pkgarch, value): + """Returns whether the specified value is found in the database for the specified package and architecture""" + + # Just returns the value if found or None otherwise + data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, + (version, pkgarch, value)) + row=data.fetchone() + if row is not None: + return True + else: + return False + + def find_value(self, version, pkgarch, checksum): + """Returns the value for the specified checksum if found or None otherwise.""" + + data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, + (version, pkgarch, checksum)) + row=data.fetchone() + if row is not None: + return row[0] + else: + return None + + def find_max_value(self, version, pkgarch): + """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value""" + + data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), + (version, pkgarch)) + row = data.fetchone() + if row is not None: + return row[0] + else: + return None + def _get_value_hist(self, version, pkgarch, checksum): data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index efb2e0cf93..86bd3bb75f 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -26,6 +26,9 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): self.handlers.update({ "get-pr": self.handle_get_pr, + "test-pr": self.handle_test_pr, + "test-package": self.handle_test_package, + "max-package-pr": self.handle_max_package_pr, "import-one": self.handle_import_one, "export": self.handle_export, "is-readonly": self.handle_is_readonly, @@ -43,6 +46,31 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): else: self.server.table.sync_if_dirty() + async def handle_test_pr(self, request): + '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value''' + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] + + value = self.server.table.find_value(version, pkgarch, checksum) + return {"value": value} + + async def handle_test_package(self, request): + '''Tells whether there are entries for (version, pkgarch) in the db. Returns True or False''' + version = request["version"] + pkgarch = request["pkgarch"] + + value = self.server.table.test_package(version, pkgarch) + return {"value": value} + + async def handle_max_package_pr(self, request): + '''Finds the greatest PR value for (version, pkgarch) in the db. Returns None if no entry was found''' + version = request["version"] + pkgarch = request["pkgarch"] + + value = self.server.table.find_max_value(version, pkgarch) + return {"value": value} + async def handle_get_pr(self, request): version = request["version"] pkgarch = request["pkgarch"] From patchwork Fri Apr 12 09:02:30 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42254 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id BDA74C0650F for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web10.42607.1712912575387518441 for ; Fri, 12 Apr 2024 02:02:55 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=VqpH6Zur; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id CF71BC0006; Fri, 12 Apr 2024 09:02:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912574; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=QWt+p3BwCcbfO+hapVrTtEEopipqQ03zbPb4MEfd72g=; b=VqpH6ZurgOEGMvjLj8fgtES05ell2Rqdpgesdv9v7OMox6O80IiAuMHS34e59DvjoCl3DQ lmRzKA7V2P9nS5P9uPIjVbvV/AmmXM4YwAqALvUK1SwYP8d/LkeAerFovNpBlThgV5hDwK Sfqmq7kHEdcOznCv95mFNxLvMWLQKJIrppguy+I7AseStCNfXNXX5B1OXvqXpONIQbOSo/ Ic9UZtBdrDffgxsyxtHDPZmuFB0Q1V7IXlylNfpktuuL3dbgRALWE0WxjE3NwLUg01le+0 gVSZZko02FmAt0MZ50dtoTQKFR/Forvkd5t49IvT8CF+5G0BNis/1jv1iR0baA== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 08/12] prserv: remove redundant exception handler Date: Fri, 12 Apr 2024 11:02:30 +0200 Message-Id: <20240412090234.4110915-9-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16083 From: Michael Opdenacker This exception handler is already present in db.py's get_value() code. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/prserv/serv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 86bd3bb75f..d1ce74e066 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -82,8 +82,6 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = {"value": value} except prserv.NotFoundError: self.logger.error("can not find value for (%s, %s)",version, checksum) - except sqlite3.Error as exc: - self.logger.error(str(exc)) return response From patchwork Fri Apr 12 09:02:31 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42256 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 924BCC05023 for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web11.42182.1712912575775805837 for ; Fri, 12 Apr 2024 02:02:56 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=WUnWOu0t; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 4DEE81BF206; Fri, 12 Apr 2024 09:02:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912574; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=BmltSZ8h2YgE+k6gGFI6Hsz3R/QXFiZUGWOT2Dp1QsU=; b=WUnWOu0tJoemaqo9in2VTnH6vBcIlOWPrB8Aber9tX/MvEQ8Zckw10/lmsnG41yTpd1i8u tG09MOyQgEd5NdHQFLJKtBFNgtC9mLaNC7bkkLkQAC6Hk+QHyJ39cGookPZLtejQ/Zqe9H 5savbJjrW8a1txWk9V8iVrl7iYZkA539inQbQ3mILOdXGPGJwt4R3MvbJb9/UJ1E6jboHF 0I2EHzUNjb1uggRp6bHXYgG5E/NgRc25bWf6pQ+N7BtYt7ABag8p9rrFk/f/R505Sdv71L YKz6TcwfAjJwna/9R+gCzu3kItSFNgfuTFU7+b5MOV6IYOXC1SB6anRLmUFGGw== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 09/12] prserv: correct error message Date: Fri, 12 Apr 2024 11:02:31 +0200 Message-Id: <20240412090234.4110915-10-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16084 From: Michael Opdenacker according to db.py, prserv.NotFoundError is returned here when adding a new value to the database failed Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/prserv/serv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index d1ce74e066..dc4be5b620 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -81,7 +81,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): value = self.server.table.get_value(version, pkgarch, checksum) response = {"value": value} except prserv.NotFoundError: - self.logger.error("can not find value for (%s, %s)",version, checksum) + self.logger.error("failure storing value in database for (%s, %s)",version, checksum) return response From patchwork Fri Apr 12 09:02:32 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42255 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 89B0DC46467 for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay7-d.mail.gandi.net (relay7-d.mail.gandi.net [217.70.183.200]) by mx.groups.io with SMTP id smtpd.web11.42184.1712912577130436861 for ; Fri, 12 Apr 2024 02:02:57 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=NuiF9REa; spf=pass (domain: bootlin.com, ip: 217.70.183.200, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 08EEA2000E; Fri, 12 Apr 2024 09:02:54 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912575; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=AnS/XMM2EAxboZahss1+koAN4pt6RPYMMv3S5FXrEdo=; b=NuiF9REa/GikVWxaSDRyiLtWLFn1lkxl7paK9B4Tc1zZYDpYQxtMDAW9jgxIHIf2uIEU+r aYGY7XJ74PlutloRC6xwKxJi9aeyw86L2mCPs7Lz0xzAgjMaMpKcHK1RZfejlCuUn1N5xG qic9DwhL1QrLElKVeCfHqdI2XIPpVxNeOLoa9RA7RbSsM1ulFOSeCEbV7IBUoPvt/djtzf gJPUETSS2EjDtZZd77urDpSsP6C66fZQfes4OXKzOrrX8FZIp5I45e5gGxSXpVEox2v7nV bGdKcnIM65uOa+UN6DeX0/TpSid/EcwgfSmrP2Lseq82rkL9m9anjd55S9rA6Q== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 10/12] prserv: remove unnecessary code Date: Fri, 12 Apr 2024 11:02:32 +0200 Message-Id: <20240412090234.4110915-11-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16085 From: Michael Opdenacker In db.py, the ifnull() statement guarantees that the SQL request will return a value. It's therefore unnecessary to test the case when no value is found. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- lib/prserv/db.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 7c200602ed..eb41508198 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -172,11 +172,7 @@ class PRTable(object): if self.read_only: data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) - row = data.fetchone() - if row is not None: - return row[0] - else: - return 0 + return data.fetchone()[0] try: self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" From patchwork Fri Apr 12 09:02:33 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42259 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 79910C04FFF for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web10.42609.1712912577043262353 for ; Fri, 12 Apr 2024 02:02:57 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=c8lKqdxi; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 7E259C0010; Fri, 12 Apr 2024 09:02:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912575; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=cMYfI3x5l/gpOE9SBMFMiDZvHzxxSZsy1c9Q4p7H7ZQ=; b=c8lKqdxi51dBx4RVjNjNzkmghsOokdbRPk29dE3E757Low7S4Y6SUtpueKYDMDtn1sfdnG fDK1dSUmrfdr/JtM4lenLg5tcoAwSn2BXdJMlAojdid4/+e+e1K5uCfYYhGe3FbvIQpDmw ADw9xBqcMb9aZxnYQUCyjTVmzMMwraHzZ3rjz8OE+svkDKu30ncsu9Nq2xZZ7CAOc3gWIG 4Te6wrBCEvDrfjUkEdQcHMfzE8JjxctIl3na5kUGhggFIi7OX8WKxHSGwJo4zoaigmP/53 WFjDwuzVbg+DeFq096xAhZ8IJ7ARotfV5v6JVQNUVSsjbTSG1lL/lMS7kRis/A== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling , Thomas Petazzoni Subject: [PATCH 11/12] prserv: add "upstream" server support Date: Fri, 12 Apr 2024 11:02:33 +0200 Message-Id: <20240412090234.4110915-12-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16086 From: Michael Opdenacker Introduce a PRSERVER_UPSTREAM variable that makes the local PR server connect to an "upstream" one. This makes it possible to implement local fixes to an upstream package (revision "x", in a way that gives the local update priority (revision "x.y"). Update the calculation of the new revisions to support the case when prior revisions are not integers, but have an "x.y..." format." Set the comments in the handle_get_pr() function in serv.py for details about the calculation of the local revision. Use the newly developed functions to simplify the implementation of the get_value() function by merging the code of the earlier _get_value_hist() and _get_value_nohist() functions, removing redundancy between both functions. This also makes the code easier to understand. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling Cc: Thomas Petazzoni --- bin/bitbake-prserv | 15 ++++- lib/prserv/__init__.py | 15 +++++ lib/prserv/client.py | 1 + lib/prserv/db.py | 125 ++++++++++++++++++++--------------------- lib/prserv/serv.py | 95 +++++++++++++++++++++++++++---- 5 files changed, 174 insertions(+), 77 deletions(-) diff --git a/bin/bitbake-prserv b/bin/bitbake-prserv index ad0a069401..e39d0fba87 100755 --- a/bin/bitbake-prserv +++ b/bin/bitbake-prserv @@ -70,12 +70,25 @@ def main(): action="store_true", help="open database in read-only mode", ) + parser.add_argument( + "-u", + "--upstream", + default=os.environ.get("PRSERVER_UPSTREAM", None), + help="Upstream PR service (host:port)", + ) args = parser.parse_args() prserv.init_logger(os.path.abspath(args.log), args.loglevel) if args.start: - ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only) + ret=prserv.serv.start_daemon( + args.file, + args.host, + args.port, + os.path.abspath(args.log), + args.read_only, + args.upstream + ) elif args.stop: ret=prserv.serv.stop_daemon(args.host, args.port) else: diff --git a/lib/prserv/__init__.py b/lib/prserv/__init__.py index 0e0aa34d0e..2ee6a28c04 100644 --- a/lib/prserv/__init__.py +++ b/lib/prserv/__init__.py @@ -8,6 +8,7 @@ __version__ = "1.0.0" import os, time import sys, logging +from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS def init_logger(logfile, loglevel): numeric_level = getattr(logging, loglevel.upper(), None) @@ -18,3 +19,17 @@ def init_logger(logfile, loglevel): class NotFoundError(Exception): pass + +async def create_async_client(addr): + from . import client + + c = client.PRAsyncClient() + + try: + (typ, a) = parse_address(addr) + await c.connect_tcp(*a) + return c + + except Exception as e: + await c.close() + raise e diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 8471ee3046..89760b6f74 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -6,6 +6,7 @@ import logging import bb.asyncrpc +from . import create_async_client logger = logging.getLogger("BitBake.PRserv") diff --git a/lib/prserv/db.py b/lib/prserv/db.py index eb41508198..8305238f7a 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -21,6 +21,20 @@ sqlversion = sqlite3.sqlite_version_info if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): raise Exception("sqlite3 version 3.3.0 or later is required.") +def increase_revision(ver): + """Take a revision string such as "1" or "1.2.3" or even a number and increase its last number + This fails if the last number is not an integer""" + + fields=str(ver).split('.') + last = fields[-1] + + try: + val = int(last) + except Exception as e: + logger.critical("Unable to increase revision value %s: %s" % (ver, e)) + + return ".".join(fields[0:-1] + list(str(val + 1))) + # # "No History" mode - for a given query tuple (version, pkgarch, checksum), # the returned value will be the largest among all the values of the same @@ -53,7 +67,7 @@ class PRTable(object): (version TEXT NOT NULL, \ pkgarch TEXT NOT NULL, \ checksum TEXT NOT NULL, \ - value INTEGER, \ + value TEXT, \ PRIMARY KEY (version, pkgarch, checksum));" % self.table) def _execute(self, *query): @@ -119,84 +133,67 @@ class PRTable(object): data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) row = data.fetchone() - if row is not None: + # With SELECT max() requests, you have an empty row when there are no values, therefore the test on row[0] + if row is not None and row[0] is not None: return row[0] else: return None - def _get_value_hist(self, version, pkgarch, checksum): - data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, - (version, pkgarch, checksum)) - row=data.fetchone() - if row is not None: - return row[0] + def find_new_subvalue(self, version, pkgarch, base): + """Take and increase the greatest ".y" value for (version, pkgarch), or return ".1" if not found. + This doesn't store a new value.""" + + data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base), + (version, pkgarch)) + row = data.fetchone() + # With SELECT max() requests, you have an empty row when there are no values, therefore the test on row[0] + if row is not None and row[0] is not None: + return increase_revision(row[0]) else: - #no value found, try to insert - if self.read_only: - data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), - (version, pkgarch)) - row = data.fetchone() - if row is not None: - return row[0] - else: - return 0 + return base + ".0" - try: - self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" - % (self.table, self.table), - (version, pkgarch, checksum, version, pkgarch)) - except sqlite3.IntegrityError as exc: - logger.error(str(exc)) + def store_value(self, version, pkgarch, checksum, value): + """Store new value in the database""" - self.dirty = True + try: + self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), + (version, pkgarch, checksum, value)) + except sqlite3.IntegrityError as exc: + logger.error(str(exc)) - data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, - (version, pkgarch, checksum)) - row=data.fetchone() - if row is not None: - return row[0] - else: - raise prserv.NotFoundError + self.dirty = True - def _get_value_no_hist(self, version, pkgarch, checksum): - data=self._execute("SELECT value FROM %s \ - WHERE version=? AND pkgarch=? AND checksum=? AND \ - value >= (select max(value) from %s where version=? AND pkgarch=?);" - % (self.table, self.table), - (version, pkgarch, checksum, version, pkgarch)) - row=data.fetchone() - if row is not None: - return row[0] - else: - #no value found, try to insert - if self.read_only: - data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), - (version, pkgarch)) - return data.fetchone()[0] + def _get_value(self, version, pkgarch, checksum): - try: - self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" - % (self.table, self.table), - (version, pkgarch, checksum, version, pkgarch)) - except sqlite3.IntegrityError as exc: - logger.error(str(exc)) - self.conn.rollback() + max_value = self.find_max_value(version, pkgarch) - self.dirty = True + if max_value is None: + # version, pkgarch completely unknown. Return initial value. + return "0" - data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, - (version, pkgarch, checksum)) - row=data.fetchone() - if row is not None: - return row[0] - else: - raise prserv.NotFoundError + value = self.find_value(version, pkgarch, checksum) + + if value is None: + # version, pkgarch found but not checksum. Create a new value from the maximum one + return increase_revision(max_value) - def get_value(self, version, pkgarch, checksum): if self.nohist: - return self._get_value_no_hist(version, pkgarch, checksum) + # "no-history" mode: only return a value if that's the maximum one for + # the version and architecture, otherwise create a new one. + # This means that the value cannot decrement. + if value == max_value: + return value + else: + return increase_revision(max_value) else: - return self._get_value_hist(version, pkgarch, checksum) + # "hist" mode: we found an existing value. We can return it + # whether it's the maximum one or not. + return value + + def get_value(self, version, pkgarch, checksum): + value = self._get_value(version, pkgarch, checksum) + self.store_value(version, pkgarch, checksum, value) + return value def _import_hist(self, version, pkgarch, checksum, value): if self.read_only: diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index dc4be5b620..9e07a34445 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -12,6 +12,7 @@ import sqlite3 import prserv import prserv.db import errno +from . import create_async_client import bb.asyncrpc logger = logging.getLogger("BitBake.PRserv") @@ -76,14 +77,76 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): pkgarch = request["pkgarch"] checksum = request["checksum"] - response = None - try: + if self.upstream_client is None: value = self.server.table.get_value(version, pkgarch, checksum) - response = {"value": value} - except prserv.NotFoundError: - self.logger.error("failure storing value in database for (%s, %s)",version, checksum) + return {"value": value} - return response + # We have an upstream server. + # Check whether the local server already knows the requested configuration + # Here we use find_value(), not get_value(), because we don't want + # to unconditionally add a new generated value to the database. If the configuration + # is a new one, the generated value we will add will depend on what's on the upstream server. + + value = self.server.table.find_value(version, pkgarch, checksum) + + if value is not None: + + # The configuration is already known locally. Let's use it. + + return {"value": value} + + # The configuration is a new one for the local server + # Let's ask the upstream server whether it knows it + + known_upstream = await self.upstream_client.test_package(version, pkgarch) + + if not known_upstream: + + # The package is not known upstream, must be a local-only package + # Let's compute the PR number using the local-only method + + value = self.server.table.get_value(version, pkgarch, checksum) + return {"value": value} + + # The package is known upstream, let's ask the upstream server + # whether it knows our new output hash + + value = await self.upstream_client.test_pr(version, pkgarch, checksum) + + if value is not None: + + # Upstream knows this output hash, let's store it and use it too. + + if not self.server.read_only: + self.server.table.store_value(version, pkgarch, checksum, value) + # If the local server is read only, won't be able to store the new + # value in the database and will have to keep asking the upstream server + + return {"value": value} + + # The output hash doesn't exist upstream, get the most recent number from upstream (x) + # Then, we want to have a new PR value for the local server: x.y + + upstream_max = await self.upstream_client.max_package_pr(version, pkgarch) + # Here we know that the package is known upstream, so upstream_max can't be None + subvalue = self.server.table.find_new_subvalue(version, pkgarch, upstream_max) + + if not self.server.read_only: + self.server.table.store_value(version, pkgarch, checksum, subvalue) + + return {"value": subvalue} + + async def process_requests(self): + if self.server.upstream is not None: + self.upstream_client = await create_async_client(self.server.upstream) + else: + self.upstream_client = None + + try: + await super().process_requests() + finally: + if self.upstream_client is not None: + await self.upstream_client.close() async def handle_import_one(self, request): response = None @@ -117,11 +180,12 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): return {"readonly": self.server.read_only} class PRServer(bb.asyncrpc.AsyncServer): - def __init__(self, dbfile, read_only=False): + def __init__(self, dbfile, read_only=False, upstream=None): super().__init__(logger) self.dbfile = dbfile self.table = None self.read_only = read_only + self.upstream = upstream def accept_client(self, socket): return PRServerClient(socket, self) @@ -134,6 +198,9 @@ class PRServer(bb.asyncrpc.AsyncServer): self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % (self.dbfile, self.address, str(os.getpid()))) + if self.upstream is not None: + self.logger.info("And upstream PRServer: %s " % (self.upstream)) + return tasks async def stop(self): @@ -147,14 +214,15 @@ class PRServer(bb.asyncrpc.AsyncServer): self.table.sync() class PRServSingleton(object): - def __init__(self, dbfile, logfile, host, port): + def __init__(self, dbfile, logfile, host, port, upstream): self.dbfile = dbfile self.logfile = logfile self.host = host self.port = port + self.upstream = upstream def start(self): - self.prserv = PRServer(self.dbfile) + self.prserv = PRServer(self.dbfile, upstream=self.upstream) self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port) self.process = self.prserv.serve_as_process(log_level=logging.WARNING) @@ -233,7 +301,7 @@ def run_as_daemon(func, pidfile, logfile): os.remove(pidfile) os._exit(0) -def start_daemon(dbfile, host, port, logfile, read_only=False): +def start_daemon(dbfile, host, port, logfile, read_only=False, upstream=None): ip = socket.gethostbyname(host) pidfile = PIDPREFIX % (ip, port) try: @@ -249,7 +317,7 @@ def start_daemon(dbfile, host, port, logfile, read_only=False): dbfile = os.path.abspath(dbfile) def daemon_main(): - server = PRServer(dbfile, read_only=read_only) + server = PRServer(dbfile, read_only=read_only, upstream=upstream) server.start_tcp_server(ip, port) server.serve_forever() @@ -336,6 +404,9 @@ def auto_start(d): host = host_params[0].strip().lower() port = int(host_params[1]) + + upstream = d.getVar("PRSERV_UPSTREAM") or None + if is_local_special(host, port): import bb.utils cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE")) @@ -350,7 +421,7 @@ def auto_start(d): auto_shutdown() if not singleton: bb.utils.mkdirhier(cachedir) - singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port) + singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port, upstream) singleton.start() if singleton: host = singleton.host From patchwork Fri Apr 12 09:02:34 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 42253 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 72E18C4345F for ; Fri, 12 Apr 2024 09:03:02 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web10.42610.1712912577471693501 for ; Fri, 12 Apr 2024 02:02:57 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=XVpn6Ytl; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id E87561BF20F; Fri, 12 Apr 2024 09:02:55 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1712912576; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=CAVCR34fY9T4tS0P9OHNq1KAyZM7hWfTARMo3Jggx1o=; b=XVpn6YtlarXeTz6eovqiPWmo2uXgz/61OsMgFrP9tb29cPzXtRGSlhEsWRJKKEQ1kp0aAc jD0c71E0Gux2A3bi4OonQWpEv60VSH++GF3fjYowbf3TmTZocwOCE0Xc6cVPzFcL7CDL7A Gdld80dnK+AQhUe31CBchzPiXV9RSlFK4pxGSKr7q6Y1WFYYu+TMK7ObJd8M4t6LyNw73a Z2jcUnncO+J5aViFHMx/sJj8vZHtD+EdAVPbSIKHzfIaNu2wpdC4vJnnhvRCeLo1N56ucT S7dNrWkeniUdX6pqAKDFefcS0uiHccUkGwNm+Aot9JcbFlbqgQFzRwGfx1z+dg== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Thomas Petazzoni , Joshua Watt , Tim Orling Subject: [PATCH 12/12] prserv: add "history" argument to "get-pr" request Date: Fri, 12 Apr 2024 11:02:34 +0200 Message-Id: <20240412090234.4110915-13-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> References: <20240412090234.4110915-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 12 Apr 2024 09:03:02 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16087 From: Michael Opdenacker This makes it possible to test the "history" mode in the future bitbake selftests, to make sure that both "history" and "no history" modes work as expected. Signed-off-by: Michael Opdenacker Cc: Thomas Petazzoni Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/client.py | 4 ++-- lib/prserv/db.py | 16 ++++++++-------- lib/prserv/serv.py | 5 +++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 89760b6f74..389172f055 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -14,9 +14,9 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): def __init__(self): super().__init__("PRSERVICE", "1.0", logger) - async def getPR(self, version, pkgarch, checksum): + async def getPR(self, version, pkgarch, checksum, history=False): response = await self.invoke( - {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} + {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}} ) if response: return response["value"] diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 8305238f7a..d41b781622 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -163,7 +163,7 @@ class PRTable(object): self.dirty = True - def _get_value(self, version, pkgarch, checksum): + def _get_value(self, version, pkgarch, checksum, history): max_value = self.find_max_value(version, pkgarch) @@ -177,7 +177,11 @@ class PRTable(object): # version, pkgarch found but not checksum. Create a new value from the maximum one return increase_revision(max_value) - if self.nohist: + if history: + # "history" mode: we found an existing value. We can return it + # whether it's the maximum one or not. + return value + else: # "no-history" mode: only return a value if that's the maximum one for # the version and architecture, otherwise create a new one. # This means that the value cannot decrement. @@ -185,13 +189,9 @@ class PRTable(object): return value else: return increase_revision(max_value) - else: - # "hist" mode: we found an existing value. We can return it - # whether it's the maximum one or not. - return value - def get_value(self, version, pkgarch, checksum): - value = self._get_value(version, pkgarch, checksum) + def get_value(self, version, pkgarch, checksum, history): + value = self._get_value(version, pkgarch, checksum, history) self.store_value(version, pkgarch, checksum, value) return value diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 9e07a34445..f692d050b8 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -76,9 +76,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): version = request["version"] pkgarch = request["pkgarch"] checksum = request["checksum"] + history = request["history"] if self.upstream_client is None: - value = self.server.table.get_value(version, pkgarch, checksum) + value = self.server.table.get_value(version, pkgarch, checksum, history) return {"value": value} # We have an upstream server. @@ -105,7 +106,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): # The package is not known upstream, must be a local-only package # Let's compute the PR number using the local-only method - value = self.server.table.get_value(version, pkgarch, checksum) + value = self.server.table.get_value(version, pkgarch, checksum, history) return {"value": value} # The package is known upstream, let's ask the upstream server