From patchwork Fri Mar 29 14:39:45 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41645 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 94B9ACD1283 for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay4-d.mail.gandi.net (relay4-d.mail.gandi.net [217.70.183.196]) by mx.groups.io with SMTP id smtpd.web10.22.1711723223857771072 for ; Fri, 29 Mar 2024 07:40:24 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=JLUFDxQY; spf=pass (domain: bootlin.com, ip: 217.70.183.196, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id F10F4E000D; Fri, 29 Mar 2024 14:40:21 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723222; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=dcAuSysxNjBs7+HkBokLy2YPGfNy6cxGuQF67J+GqaY=; b=JLUFDxQY0BNs+0+Eyk+0+1Hub8DKt7JUZNYZkvDqx9PMLP441tUoiqR2CoeLeVcu/1Zw+B jFAJ7CwcrDEbrdt1mhkLa1+1zo33HduTU2WfRvx1lVv23sZsqSWTe+Wzkkfa6X0vzIXE+l bM+/qT7GXoC7FkqQOQyAMxSSBCdcpstr52Ik2ram+XlWAxHZLjS8G5ihm2s7TxIwGkD4Do 0ZTFu8f02TpEMVrxuaKroTNSrcRLojdZzw1UNoz1Z0JhCYIRHWCduuCOhi3OO3apQIefuD aIo0ux786QRM58YddjUtWlUaH8f3tkT2tvLtqKy/HHGct+yGhUi1O2IoXteo3g== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 01/12] prserv: simplify the PRServerClient() interface Date: Fri, 29 Mar 2024 15:39:45 +0100 Message-Id: <20240329143956.1602707-2-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16023 From: Michael Opdenacker serv.py: simplify the PRServerClient() interface by passing the server object instead of multiple arguments, and then retrieving the data through this object. This replicates what is done for ServerClient() in hashserv/server.py Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/serv.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 5fc8863f70..8b1bdc0250 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -20,16 +20,16 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid" singleton = None class PRServerClient(bb.asyncrpc.AsyncServerConnection): - def __init__(self, socket, table, read_only): - super().__init__(socket, 'PRSERVICE', logger) + def __init__(self, socket, server): + super().__init__(socket, 'PRSERVICE', server.logger) + self.server = server + self.handlers.update({ 'get-pr': self.handle_get_pr, 'import-one': self.handle_import_one, 'export': self.handle_export, 'is-readonly': self.handle_is_readonly, }) - self.table = table - self.read_only = read_only def validate_proto_version(self): return (self.proto_version == (1, 0)) @@ -38,10 +38,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): try: return await super().dispatch_message(msg) except: - self.table.sync() + self.server.table.sync() raise else: - self.table.sync_if_dirty() + self.server.table.sync_if_dirty() async def handle_get_pr(self, request): version = request['version'] @@ -50,7 +50,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = None try: - value = self.table.getValue(version, pkgarch, checksum) + value = self.server.table.getValue(version, pkgarch, checksum) response = {'value': value} except prserv.NotFoundError: logger.error("can not find value for (%s, %s)",version, checksum) @@ -67,7 +67,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): checksum = request['checksum'] value = request['value'] - value = self.table.importone(version, pkgarch, checksum, value) + value = self.server.table.importone(version, pkgarch, checksum, value) if value is not None: response = {'value': value} @@ -80,7 +80,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): colinfo = request['colinfo'] try: - (metainfo, datainfo) = self.table.export(version, pkgarch, checksum, colinfo) + (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) except sqlite3.Error as exc: logger.error(str(exc)) metainfo = datainfo = None @@ -88,7 +88,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): return {'metainfo': metainfo, 'datainfo': datainfo} async def handle_is_readonly(self, request): - return {'readonly': self.read_only} + return {'readonly': self.server.read_only} class PRServer(bb.asyncrpc.AsyncServer): def __init__(self, dbfile, read_only=False): @@ -98,7 +98,7 @@ class PRServer(bb.asyncrpc.AsyncServer): self.read_only = read_only def accept_client(self, socket): - return PRServerClient(socket, self.table, self.read_only) + return PRServerClient(socket, self) def start(self): tasks = super().start() From patchwork Fri Mar 29 14:39:46 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41651 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A279BCD128D for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay3-d.mail.gandi.net (relay3-d.mail.gandi.net [217.70.183.195]) by mx.groups.io with SMTP id smtpd.web11.24.1711723224594854424 for ; Fri, 29 Mar 2024 07:40:25 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=CIRRFbBL; spf=pass (domain: bootlin.com, ip: 217.70.183.195, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 9D2466000A; Fri, 29 Mar 2024 14:40:22 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723222; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=Gk2MT6ALXSNZC0oA0Fc4TEPpn/zjL+x3k7mpM+0C/Zk=; b=CIRRFbBLoZSZMX7N8StKMuc5I8wpKvGv0ry+XNNl3KfU5jfmzrvgUxRCyPuIVbAP0SABbl n6Tdvla1cx0NI8KqewNz+lYzLB4KZXTdpB3MeQXmIzdPEBBnsljXPuGFfQCY7B2Ib8xuQe Hl3jd0jioFcIK55x0Ien34xkIKoFz2+C5LUqAMkopiJNr19TnzA2VTMvg7aJCnhjt6dMto c39aKJRH+uk8ZYMeWAIa42MP0wDku0/NUhH6tzt3rmf4eR7BPZvmBpGUxGKGW2HAin0ShE oJFkgdISme34riR8ond9LiLLZe7MqQ6eQ0JFLy6FI85yGDN9+C1CZgNM8/4dEQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 02/12] prserv: use double quotes by default Date: Fri, 29 Mar 2024 15:39:46 +0100 Message-Id: <20240329143956.1602707-3-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16024 From: Michael Opdenacker To aligh with the hashserv code Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- bin/bitbake-prserv | 4 +-- lib/prserv/__init__.py | 4 +-- lib/prserv/client.py | 20 ++++++------- lib/prserv/db.py | 32 ++++++++++----------- lib/prserv/serv.py | 64 +++++++++++++++++++++--------------------- 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/bin/bitbake-prserv b/bin/bitbake-prserv index 5be42f3ce5..8c3808fb20 100755 --- a/bin/bitbake-prserv +++ b/bin/bitbake-prserv @@ -11,14 +11,14 @@ import optparse import warnings warnings.simplefilter("default") -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib")) import prserv import prserv.serv __version__="1.0.0" -PRHOST_DEFAULT='0.0.0.0' +PRHOST_DEFAULT="0.0.0.0" PRPORT_DEFAULT=8585 def main(): diff --git a/lib/prserv/__init__.py b/lib/prserv/__init__.py index 38ced818ad..5790a8db1e 100644 --- a/lib/prserv/__init__.py +++ b/lib/prserv/__init__.py @@ -12,8 +12,8 @@ import sys,logging def init_logger(logfile, loglevel): numeric_level = getattr(logging, loglevel.upper(), None) if not isinstance(numeric_level, int): - raise ValueError('Invalid log level: %s' % loglevel) - FORMAT = '%(asctime)-15s %(message)s' + raise ValueError("Invalid log level: %s" % loglevel) + FORMAT = "%(asctime)-15s %(message)s" logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT) class NotFoundError(Exception): diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 6b81356fac..7bc5188c53 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -11,40 +11,40 @@ logger = logging.getLogger("BitBake.PRserv") class PRAsyncClient(bb.asyncrpc.AsyncClient): def __init__(self): - super().__init__('PRSERVICE', '1.0', logger) + super().__init__("PRSERVICE", "1.0", logger) async def getPR(self, version, pkgarch, checksum): response = await self.invoke( - {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}} + {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} ) if response: - return response['value'] + return response["value"] async def importone(self, version, pkgarch, checksum, value): response = await self.invoke( - {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}} + {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}} ) if response: - return response['value'] + return response["value"] async def export(self, version, pkgarch, checksum, colinfo): response = await self.invoke( - {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}} + {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo}} ) if response: - return (response['metainfo'], response['datainfo']) + return (response["metainfo"], response["datainfo"]) async def is_readonly(self): response = await self.invoke( - {'is-readonly': {}} + {"is-readonly": {}} ) if response: - return response['readonly'] + return response["readonly"] class PRClient(bb.asyncrpc.Client): def __init__(self): super().__init__() - self._add_methods('getPR', 'importone', 'export', 'is_readonly') + self._add_methods("getPR", "importone", "export", "is_readonly") def _get_async_client(self): return PRAsyncClient() diff --git a/lib/prserv/db.py b/lib/prserv/db.py index b4bda7078c..0859cf4f2c 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -64,7 +64,7 @@ class PRTable(object): try: return self.conn.execute(*query) except sqlite3.OperationalError as exc: - if 'is locked' in str(exc) and end > time.time(): + if "is locked" in str(exc) and end > time.time(): continue raise exc @@ -220,18 +220,18 @@ class PRTable(object): metainfo = {} #column info if colinfo: - metainfo['tbl_name'] = self.table - metainfo['core_ver'] = prserv.__version__ - metainfo['col_info'] = [] + metainfo["tbl_name"] = self.table + metainfo["core_ver"] = prserv.__version__ + metainfo["col_info"] = [] data = self._execute("PRAGMA table_info(%s);" % self.table) for row in data: col = {} - col['name'] = row['name'] - col['type'] = row['type'] - col['notnull'] = row['notnull'] - col['dflt_value'] = row['dflt_value'] - col['pk'] = row['pk'] - metainfo['col_info'].append(col) + col["name"] = row["name"] + col["type"] = row["type"] + col["notnull"] = row["notnull"] + col["dflt_value"] = row["dflt_value"] + col["pk"] = row["pk"] + metainfo["col_info"].append(col) #data info datainfo = [] @@ -261,12 +261,12 @@ class PRTable(object): else: data = self._execute(sqlstmt) for row in data: - if row['version']: + if row["version"]: col = {} - col['version'] = row['version'] - col['pkgarch'] = row['pkgarch'] - col['checksum'] = row['checksum'] - col['value'] = row['value'] + col["version"] = row["version"] + col["pkgarch"] = row["pkgarch"] + col["checksum"] = row["checksum"] + col["value"] = row["value"] datainfo.append(col) return (metainfo, datainfo) @@ -275,7 +275,7 @@ class PRTable(object): for line in self.conn.iterdump(): writeCount = writeCount + len(line) + 1 fd.write(line) - fd.write('\n') + fd.write("\n") return writeCount class PRData(object): diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 8b1bdc0250..874edd7069 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -21,14 +21,14 @@ singleton = None class PRServerClient(bb.asyncrpc.AsyncServerConnection): def __init__(self, socket, server): - super().__init__(socket, 'PRSERVICE', server.logger) + super().__init__(socket, "PRSERVICE", server.logger) self.server = server self.handlers.update({ - 'get-pr': self.handle_get_pr, - 'import-one': self.handle_import_one, - 'export': self.handle_export, - 'is-readonly': self.handle_is_readonly, + "get-pr": self.handle_get_pr, + "import-one": self.handle_import_one, + "export": self.handle_export, + "is-readonly": self.handle_is_readonly, }) def validate_proto_version(self): @@ -44,14 +44,14 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): self.server.table.sync_if_dirty() async def handle_get_pr(self, request): - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] response = None try: value = self.server.table.getValue(version, pkgarch, checksum) - response = {'value': value} + response = {"value": value} except prserv.NotFoundError: logger.error("can not find value for (%s, %s)",version, checksum) except sqlite3.Error as exc: @@ -62,22 +62,22 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): async def handle_import_one(self, request): response = None if not self.read_only: - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] - value = request['value'] + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] + value = request["value"] value = self.server.table.importone(version, pkgarch, checksum, value) if value is not None: - response = {'value': value} + response = {"value": value} return response async def handle_export(self, request): - version = request['version'] - pkgarch = request['pkgarch'] - checksum = request['checksum'] - colinfo = request['colinfo'] + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] + colinfo = request["colinfo"] try: (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) @@ -85,10 +85,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): logger.error(str(exc)) metainfo = datainfo = None - return {'metainfo': metainfo, 'datainfo': datainfo} + return {"metainfo": metainfo, "datainfo": datainfo} async def handle_is_readonly(self, request): - return {'readonly': self.server.read_only} + return {"readonly": self.server.read_only} class PRServer(bb.asyncrpc.AsyncServer): def __init__(self, dbfile, read_only=False): @@ -135,7 +135,7 @@ class PRServSingleton(object): if not self.prserv.address: raise PRServiceConfigError if not self.port: - self.port = int(self.prserv.address.rsplit(':', 1)[1]) + self.port = int(self.prserv.address.rsplit(":", 1)[1]) def run_as_daemon(func, pidfile, logfile): """ @@ -171,12 +171,12 @@ def run_as_daemon(func, pidfile, logfile): # stdout/stderr or it could be 'real' unix fd forking where we need # to physically close the fds to prevent the program launching us from # potentially hanging on a pipe. Handle both cases. - si = open('/dev/null', 'r') + si = open("/dev/null", "r") try: os.dup2(si.fileno(),sys.stdin.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stdin = si - so = open(logfile, 'a+') + so = open(logfile, "a+") try: os.dup2(so.fileno(),sys.stdout.fileno()) except (AttributeError, io.UnsupportedOperation): @@ -200,7 +200,7 @@ def run_as_daemon(func, pidfile, logfile): # write pidfile pid = str(os.getpid()) - with open(pidfile, 'w') as pf: + with open(pidfile, "w") as pf: pf.write("%s\n" % pid) func() @@ -245,12 +245,12 @@ def stop_daemon(host, port): # so at least advise the user which ports the corresponding server is listening ports = [] portstr = "" - for pf in glob.glob(PIDPREFIX % (ip,'*')): + for pf in glob.glob(PIDPREFIX % (ip, "*")): bn = os.path.basename(pf) root, _ = os.path.splitext(bn) - ports.append(root.split('_')[-1]) + ports.append(root.split("_")[-1]) if len(ports): - portstr = "Wrong port? Other ports listening at %s: %s" % (host, ' '.join(ports)) + portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports)) sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n" % (pidfile,portstr)) @@ -284,7 +284,7 @@ def is_running(pid): return True def is_local_special(host, port): - if (host == 'localhost' or host == '127.0.0.1') and not port: + if (host == "localhost" or host == "127.0.0.1") and not port: return True else: return False @@ -295,7 +295,7 @@ class PRServiceConfigError(Exception): def auto_start(d): global singleton - host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':'))) + host_params = list(filter(None, (d.getVar("PRSERV_HOST") or "").split(":"))) if not host_params: # Shutdown any existing PR Server auto_shutdown() @@ -304,7 +304,7 @@ def auto_start(d): if len(host_params) != 2: # Shutdown any existing PR Server auto_shutdown() - logger.critical('\n'.join(['PRSERV_HOST: incorrect format', + logger.critical("\n".join(["PRSERV_HOST: incorrect format", 'Usage: PRSERV_HOST = ":"'])) raise PRServiceConfigError @@ -357,8 +357,8 @@ def connect(host, port): global singleton - if host.strip().lower() == 'localhost' and not port: - host = 'localhost' + if host.strip().lower() == "localhost" and not port: + host = "localhost" port = singleton.port conn = client.PRClient() From patchwork Fri Mar 29 14:39:47 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41648 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id C7E29CD1291 for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web11.25.1711723225272335395 for ; Fri, 29 Mar 2024 07:40:25 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=fc0blRhq; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 4DEE7C0005; Fri, 29 Mar 2024 14:40:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723223; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=xHnx3AYMKTLPfPEJ1mMEUn5TKsr/3wVscnGR7vucOkI=; b=fc0blRhqQaaaQCZrAXsxwvEdhfsQU+JgncPB1VSVIcj+/WquASHCc5qomENugXXWXTNPUL oPDtw5aGMGVnVAXK9ynfS8qPS3PUiMPrtOyfAYz/Qn21Zpnil5EAoEfpdgyGwhCNFizc0y Xz/oBRXFMNpfsuQFl0W0QFBFiHNX17r4RVjVRT2UWmkd3LpSizJA/QKlLGhFgiWfHsoeAu qjaIZ4MkvC5z/9fqfI8KCC/lwetPc2+7Yf1MwfWCDeQilmnKb/oMMrZQ0k09CIzsAl+hhl 48pzP5ysgukn54/sSb+pOROjNFfk3+2L1uDieVhxANYkDFHfeiflO/OTXSGwGQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 03/12] bitbake-prserv: replace deprecated optparse by argparse Date: Fri, 29 Mar 2024 15:39:47 +0100 Message-Id: <20240329143956.1602707-4-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16025 From: Michael Opdenacker optparse is deprecated since Python 2.7 Note that this is neither supposed to change the options supported by bitbake-prserv nor the way they are interpreted. Note that in the "--help" output, long options are now reported for example as "--host HOST" instead of "--host=HOST" but both are equivalent anyway, as they already were with optparse. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- bin/bitbake-prserv | 82 +++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/bin/bitbake-prserv b/bin/bitbake-prserv index 8c3808fb20..ad0a069401 100755 --- a/bin/bitbake-prserv +++ b/bin/bitbake-prserv @@ -7,7 +7,7 @@ import os import sys,logging -import optparse +import argparse import warnings warnings.simplefilter("default") @@ -16,40 +16,68 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib import prserv import prserv.serv -__version__="1.0.0" +VERSION = "1.1.0" PRHOST_DEFAULT="0.0.0.0" PRPORT_DEFAULT=8585 def main(): - parser = optparse.OptionParser( - version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__), - usage = "%prog < --start | --stop > [options]") + parser = argparse.ArgumentParser( + description="BitBake PR Server. Version=%s" % VERSION, + formatter_class=argparse.RawTextHelpFormatter) - parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store", - dest="dbfile", type="string", default="prserv.sqlite3") - parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store", - dest="logfile", type="string", default="prserv.log") - parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", - action = "store", type="string", dest="loglevel", default = "INFO") - parser.add_option("--start", help="start daemon", - action="store_true", dest="start") - parser.add_option("--stop", help="stop daemon", - action="store_true", dest="stop") - parser.add_option("--host", help="ip address to bind", action="store", - dest="host", type="string", default=PRHOST_DEFAULT) - parser.add_option("--port", help="port number(default: 8585)", action="store", - dest="port", type="int", default=PRPORT_DEFAULT) - parser.add_option("-r", "--read-only", help="open database in read-only mode", - action="store_true") + parser.add_argument( + "-f", + "--file", + default="prserv.sqlite3", + help="database filename (default: prserv.sqlite3)", + ) + parser.add_argument( + "-l", + "--log", + default="prserv.log", + help="log filename(default: prserv.log)", + ) + parser.add_argument( + "--loglevel", + default="INFO", + help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", + ) + parser.add_argument( + "--start", + action="store_true", + help="start daemon", + ) + parser.add_argument( + "--stop", + action="store_true", + help="stop daemon", + ) + parser.add_argument( + "--host", + help="ip address to bind", + default=PRHOST_DEFAULT, + ) + parser.add_argument( + "--port", + type=int, + default=PRPORT_DEFAULT, + help="port number (default: 8585)", + ) + parser.add_argument( + "-r", + "--read-only", + action="store_true", + help="open database in read-only mode", + ) - options, args = parser.parse_args(sys.argv) - prserv.init_logger(os.path.abspath(options.logfile),options.loglevel) + args = parser.parse_args() + prserv.init_logger(os.path.abspath(args.log), args.loglevel) - if options.start: - ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile), options.read_only) - elif options.stop: - ret=prserv.serv.stop_daemon(options.host, options.port) + if args.start: + ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only) + elif args.stop: + ret=prserv.serv.stop_daemon(args.host, args.port) else: ret=parser.print_help() return ret From patchwork Fri Mar 29 14:39:48 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41650 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id D6AC7CD1293 for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web11.28.1711723226172600666 for ; Fri, 29 Mar 2024 07:40:26 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=ADJgQjDh; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 3D96E1BF207; Fri, 29 Mar 2024 14:40:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723224; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=YaCtMatgb1hwvYnCGaN8RiVGv0br/058v/0hyi2SfyA=; b=ADJgQjDhV+aSMhDC08L6MGJvxeo0Bm/17nI422l7TjMoIOtdfvqwRX8FFByV7H8IsdseTY 91qcikDqjx2fKMO1jOTpExGUwluM6p3f5NMxHi8YW+4xVZ8YpiTrmkmiVTGUPYFXyc6ZtL lE5FHBaLX/uhcG+Wa7dUEyoX3EynGxzY0UyDGmRSSe4CwFE6ucNBkols/R+zIUcnV6s6fh NXMqYIWb6ZvcC2nprd2SHA/u51lJodkmCXawohbBXogLoN7NzePfXQ9PeZXgZG/kjgvhpP 4xrcEMhpC9Kp+N8+y0ZWxuxWimGC0/cYjWYTbm219zUvqivNAVECKKSgLNN0KQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 04/12] prserv: use self.logger instead of logger directly Date: Fri, 29 Mar 2024 15:39:48 +0100 Message-Id: <20240329143956.1602707-5-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16026 From: Michael Opdenacker In both the PRServerClient and PRClient objects. This aligns with what is done in hashserv/server.py and makes it possible to benefit from possible specializations of the logger in the corresponding super classes, instead of using always the global logger. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/serv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 874edd7069..eafa76673c 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -53,9 +53,9 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): value = self.server.table.getValue(version, pkgarch, checksum) response = {"value": value} except prserv.NotFoundError: - logger.error("can not find value for (%s, %s)",version, checksum) + self.logger.error("can not find value for (%s, %s)",version, checksum) except sqlite3.Error as exc: - logger.error(str(exc)) + self.logger.error(str(exc)) return response @@ -82,7 +82,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): try: (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) except sqlite3.Error as exc: - logger.error(str(exc)) + self.logger.error(str(exc)) metainfo = datainfo = None return {"metainfo": metainfo, "datainfo": datainfo} @@ -105,7 +105,7 @@ class PRServer(bb.asyncrpc.AsyncServer): self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only) self.table = self.db["PRMAIN"] - logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % + self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % (self.dbfile, self.address, str(os.getpid()))) return tasks From patchwork Fri Mar 29 14:39:49 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41646 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id A27D9CD1290 for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay5-d.mail.gandi.net (relay5-d.mail.gandi.net [217.70.183.197]) by mx.groups.io with SMTP id smtpd.web11.30.1711723226900425625 for ; Fri, 29 Mar 2024 07:40:27 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=iRBZlO0r; spf=pass (domain: bootlin.com, ip: 217.70.183.197, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id DAB491C0004; Fri, 29 Mar 2024 14:40:24 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723225; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=kIqlDTmbuOku3ZrgLpiHPjTlKJMFSBs+xT4l136LKwo=; b=iRBZlO0rClYSW9J+w40ja/E90zn1oKJb1wYF/8be6ggPpDGIzPHTl4TPfjxMxc8c4aBG8g 8t4vLh0+Z14i10Z8ew6TLoE0OVo9GH+mCYvEMMVZS4yoKRopI3JZ8KeFvcxA1+RsE32ccW EmhmxGmJlDZwWIG24jd5YuER+gNChikZNaswKTcNdxjfX4nUEtpftJ9JAszckEaR4FqOtB QUDfVE7VGyb1tbC7cepmzZKqZbEhKgvqw8g2OIaj5Ve8aRQL2AyBrqqZfaC5wt7e4lglwa 74dhs9z9Z+DSDsgi/ahZNlqIRxE3fDJu4ndrCB+Deyxz9escB9YQM+Is4Id50A== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 05/12] asyncrpc: include parse_address from hashserv Date: Fri, 29 Mar 2024 15:39:49 +0100 Message-Id: <20240329143956.1602707-6-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16028 From: Michael Opdenacker Moving the code and related definitions from hashserv/__init__.py to asyncrpc/client.py, allowing this function to be used in other asyncrpc clients. Signed-off-by: Michael Opdenacker Suggested-by: Joshua Watt Cc: Tim Orling --- lib/bb/asyncrpc/client.py | 23 +++++++++++++++++++++++ lib/hashserv/__init__.py | 27 +-------------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/lib/bb/asyncrpc/client.py b/lib/bb/asyncrpc/client.py index 29a5ab76aa..a350b4fb12 100644 --- a/lib/bb/asyncrpc/client.py +++ b/lib/bb/asyncrpc/client.py @@ -10,11 +10,34 @@ import json import os import socket import sys +import re import contextlib from threading import Thread from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK from .exceptions import ConnectionClosedError, InvokeError +UNIX_PREFIX = "unix://" +WS_PREFIX = "ws://" +WSS_PREFIX = "wss://" + +ADDR_TYPE_UNIX = 0 +ADDR_TYPE_TCP = 1 +ADDR_TYPE_WS = 2 + +def parse_address(addr): + if addr.startswith(UNIX_PREFIX): + return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],)) + elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX): + return (ADDR_TYPE_WS, (addr,)) + else: + m = re.match(r"\[(?P[^\]]*)\]:(?P\d+)$", addr) + if m is not None: + host = m.group("host") + port = m.group("port") + else: + host, port = addr.split(":") + + return (ADDR_TYPE_TCP, (host, int(port))) class AsyncClient(object): def __init__( diff --git a/lib/hashserv/__init__.py b/lib/hashserv/__init__.py index 552a33278f..74367eb6b4 100644 --- a/lib/hashserv/__init__.py +++ b/lib/hashserv/__init__.py @@ -5,39 +5,14 @@ import asyncio from contextlib import closing -import re import itertools import json from collections import namedtuple from urllib.parse import urlparse - -UNIX_PREFIX = "unix://" -WS_PREFIX = "ws://" -WSS_PREFIX = "wss://" - -ADDR_TYPE_UNIX = 0 -ADDR_TYPE_TCP = 1 -ADDR_TYPE_WS = 2 +from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS User = namedtuple("User", ("username", "permissions")) - -def parse_address(addr): - if addr.startswith(UNIX_PREFIX): - return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],)) - elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX): - return (ADDR_TYPE_WS, (addr,)) - else: - m = re.match(r"\[(?P[^\]]*)\]:(?P\d+)$", addr) - if m is not None: - host = m.group("host") - port = m.group("port") - else: - host, port = addr.split(":") - - return (ADDR_TYPE_TCP, (host, int(port))) - - def create_server( addr, dbname, From patchwork Fri Mar 29 14:39:50 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41647 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id B47B8C6FD1F for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web11.29.1711723226797452240 for ; Fri, 29 Mar 2024 07:40:27 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=HSJQZWtJ; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 4E6CA1BF208; Fri, 29 Mar 2024 14:40:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723225; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=jIi0PgcRS/G7nxYlWgyv44S0FvzjWMMYTYeBCYU6K9Q=; b=HSJQZWtJfe3TiE+wyeEpg6G1jHBL31LT6fTmRzrSovTbWDIH6SHNc86iE+Y0FSjXXQxb5z miE1kzrDzjPZA8ndgkJ10pImq7geSB+Q/w4U0IfFYEu+n4Xh8jyjO/VJfBFoHNWOkKq42K vAtL2mpHEuKnqGMHR5zx2/E74xZIlvEIA4pBhSPcoX71cRK5kmUaHML1aDxbVXtXE5D/6Z x7cnedklsm799OsSOqa0Vfz/1frbFXHefiG5v+LyXJ6wCBToa2cZXTcbzarF+rVw/Hm9zS CPB6icCxUW/qywlaAy7a+QY1jdlNKrmoh/TcFjOHNYIAJNZDuVhbrinROA4AQQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 06/12] prserv: capitalization and spacing improvements Date: Fri, 29 Mar 2024 15:39:50 +0100 Message-Id: <20240329143956.1602707-7-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16027 From: Michael Opdenacker Choosing only one style of capitalization Add extra space after some commas too Remove idle spaces Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/__init__.py | 2 +- lib/prserv/db.py | 56 +++++++++++++++++++++--------------------- lib/prserv/serv.py | 10 ++++---- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/prserv/__init__.py b/lib/prserv/__init__.py index 5790a8db1e..0e0aa34d0e 100644 --- a/lib/prserv/__init__.py +++ b/lib/prserv/__init__.py @@ -7,7 +7,7 @@ __version__ = "1.0.0" import os, time -import sys,logging +import sys, logging def init_logger(logfile, loglevel): numeric_level = getattr(logging, loglevel.upper(), None) diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 0859cf4f2c..7bc2b2dc2d 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -38,9 +38,9 @@ class PRTable(object): self.read_only = read_only self.dirty = False if nohist: - self.table = "%s_nohist" % table + self.table = "%s_nohist" % table else: - self.table = "%s_hist" % table + self.table = "%s_hist" % table if self.read_only: table_exists = self._execute( @@ -78,7 +78,7 @@ class PRTable(object): self.sync() self.dirty = False - def _getValueHist(self, version, pkgarch, checksum): + def _get_value_hist(self, version, pkgarch, checksum): data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) row=data.fetchone() @@ -87,7 +87,7 @@ class PRTable(object): else: #no value found, try to insert if self.read_only: - data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), + data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) row = data.fetchone() if row is not None: @@ -96,9 +96,9 @@ class PRTable(object): return 0 try: - self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" - % (self.table,self.table), - (version,pkgarch, checksum,version, pkgarch)) + self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" + % (self.table, self.table), + (version, pkgarch, checksum, version, pkgarch)) except sqlite3.IntegrityError as exc: logger.error(str(exc)) @@ -112,10 +112,10 @@ class PRTable(object): else: raise prserv.NotFoundError - def _getValueNohist(self, version, pkgarch, checksum): + def _get_value_no_hist(self, version, pkgarch, checksum): data=self._execute("SELECT value FROM %s \ WHERE version=? AND pkgarch=? AND checksum=? AND \ - value >= (select max(value) from %s where version=? AND pkgarch=?);" + value >= (select max(value) from %s where version=? AND pkgarch=?);" % (self.table, self.table), (version, pkgarch, checksum, version, pkgarch)) row=data.fetchone() @@ -124,7 +124,7 @@ class PRTable(object): else: #no value found, try to insert if self.read_only: - data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table), + data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) row = data.fetchone() if row is not None: @@ -133,8 +133,8 @@ class PRTable(object): return 0 try: - self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" - % (self.table,self.table), + self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" + % (self.table, self.table), (version, pkgarch, checksum, version, pkgarch)) except sqlite3.IntegrityError as exc: logger.error(str(exc)) @@ -150,17 +150,17 @@ class PRTable(object): else: raise prserv.NotFoundError - def getValue(self, version, pkgarch, checksum): + def get_value(self, version, pkgarch, checksum): if self.nohist: - return self._getValueNohist(version, pkgarch, checksum) + return self._get_value_no_hist(version, pkgarch, checksum) else: - return self._getValueHist(version, pkgarch, checksum) + return self._get_value_hist(version, pkgarch, checksum) - def _importHist(self, version, pkgarch, checksum, value): + def _import_hist(self, version, pkgarch, checksum, value): if self.read_only: return None - val = None + val = None data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) row = data.fetchone() @@ -183,27 +183,27 @@ class PRTable(object): val = row[0] return val - def _importNohist(self, version, pkgarch, checksum, value): + def _import_no_hist(self, version, pkgarch, checksum, value): if self.read_only: return None try: #try to insert self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), - (version, pkgarch, checksum,value)) + (version, pkgarch, checksum, value)) except sqlite3.IntegrityError as exc: #already have the record, try to update try: - self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value=?;" % self.table, - (version,pkgarch,checksum,value)) + (version, pkgarch, checksum, value)) row=data.fetchone() if row is not None: return row[0] @@ -212,13 +212,13 @@ class PRTable(object): def importone(self, version, pkgarch, checksum, value): if self.nohist: - return self._importNohist(version, pkgarch, checksum, value) + return self._import_no_hist(version, pkgarch, checksum, value) else: - return self._importHist(version, pkgarch, checksum, value) + return self._import_hist(version, pkgarch, checksum, value) def export(self, version, pkgarch, checksum, colinfo): metainfo = {} - #column info + #column info if colinfo: metainfo["tbl_name"] = self.table metainfo["core_ver"] = prserv.__version__ @@ -238,7 +238,7 @@ class PRTable(object): if self.nohist: sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ - (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \ + (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \ WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) else: sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table @@ -302,7 +302,7 @@ class PRData(object): def disconnect(self): self.connection.close() - def __getitem__(self,tblname): + def __getitem__(self, tblname): if not isinstance(tblname, str): raise TypeError("tblname argument must be a string, not '%s'" % type(tblname)) @@ -316,4 +316,4 @@ class PRData(object): if tblname in self._tables: del self._tables[tblname] logger.info("drop table %s" % (tblname)) - self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) + self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index eafa76673c..242cd11a5a 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -50,7 +50,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = None try: - value = self.server.table.getValue(version, pkgarch, checksum) + value = self.server.table.get_value(version, pkgarch, checksum) response = {"value": value} except prserv.NotFoundError: self.logger.error("can not find value for (%s, %s)",version, checksum) @@ -173,16 +173,16 @@ def run_as_daemon(func, pidfile, logfile): # potentially hanging on a pipe. Handle both cases. si = open("/dev/null", "r") try: - os.dup2(si.fileno(),sys.stdin.fileno()) + os.dup2(si.fileno(), sys.stdin.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stdin = si so = open(logfile, "a+") try: - os.dup2(so.fileno(),sys.stdout.fileno()) + os.dup2(so.fileno(), sys.stdout.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stdout = so try: - os.dup2(so.fileno(),sys.stderr.fileno()) + os.dup2(so.fileno(), sys.stderr.fileno()) except (AttributeError, io.UnsupportedOperation): sys.stderr = so @@ -253,7 +253,7 @@ def stop_daemon(host, port): portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports)) sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n" - % (pidfile,portstr)) + % (pidfile, portstr)) return 1 try: From patchwork Fri Mar 29 14:39:51 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41644 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 938C1CD11DD for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay6-d.mail.gandi.net (relay6-d.mail.gandi.net [217.70.183.198]) by mx.groups.io with SMTP id smtpd.web11.31.1711723227243929616 for ; Fri, 29 Mar 2024 07:40:27 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=loDtvtRm; spf=pass (domain: bootlin.com, ip: 217.70.183.198, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id B32B3C0004; Fri, 29 Mar 2024 14:40:25 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723225; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=rjad1F2DlqYmiwAQ5co92HERFnqOdXslsv9F3W0fQ80=; b=loDtvtRmCgXGw8HnF07JRIOqbjeQBia6cdhSdBF4r4EXUkyg3IMX4w42j+zrCjuzYQJX3A M4BSfC4b5K5yztChBMS7iggc00hobyhDDo14T86ThnHqPQwTmQQUW7UU5KT19Z2SM7utDy v9u5fAbor97phnm1s++aE1nQ9BJBC3vkk+ObzA0FOKf8CkP6YwjKwJC9qNJ3qON5XrJziS 3W0a1wIC7UzoHE78CFbiDJh3L8EtYOgdj11+sEfhlPGucINYzQYVxw0U5VtUKTiQ0q+yOA bgZW+QxZAbqZJfI5qHxSNWOiIvtKWJ85hFY9NWZDRX/9o24bhpIgK8PCJ78Jhw== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 07/12] prserv: fix read_only test Date: Fri, 29 Mar 2024 15:39:51 +0100 Message-Id: <20240329143956.1602707-8-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16029 From: Michael Opdenacker read_only is an attribute of the parent server object, not of the client. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/serv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 242cd11a5a..efb2e0cf93 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -61,7 +61,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): async def handle_import_one(self, request): response = None - if not self.read_only: + if not self.server.read_only: version = request["version"] pkgarch = request["pkgarch"] checksum = request["checksum"] From patchwork Fri Mar 29 14:39:52 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41653 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id DE646C6FD1F for ; Fri, 29 Mar 2024 14:40:38 +0000 (UTC) Received: from relay2-d.mail.gandi.net (relay2-d.mail.gandi.net [217.70.183.194]) by mx.groups.io with SMTP id smtpd.web11.33.1711723228392570366 for ; Fri, 29 Mar 2024 07:40:28 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=PTkCyWwn; spf=pass (domain: bootlin.com, ip: 217.70.183.194, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 6CE7840007; Fri, 29 Mar 2024 14:40:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723226; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=4as4J5FGN4ALjb2DmyYWNX4DfULaVSrsMwFglLUpe14=; b=PTkCyWwnYRIIqDAGL8ULmm5H2RSgVUQBb7hxquQzfmGvksv/cyMge1mD/ZES2haXvpX7QU mp0JBa6JHZ+l5p7ERJIBGgoVMlpDiJcqXpeUkS7jaG8sSVB/f0R0WPdYIoo1e840DPU0go gVg1PG3MkWGaqMnFm8enBWylT74+Yz7lPSDJJLd1d7XhCbNG7QebBJB26HYG0Djk3oCgIw 69kC4hLxxok0HoAmpVeSczC73qBBu3CnRnMwQq6TsKkF4CFa2c7gnXCkLPkcOfFsWhxhSy 0WhOwaNXOS6OGZJ3DmfFutCqRvJ70Sok4C2ZExVYqSs09FQkM65EYXgkfB/gaA== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 08/12] prserv: add extra requests Date: Fri, 29 Mar 2024 15:39:52 +0100 Message-Id: <20240329143956.1602707-9-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:38 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16031 From: Michael Opdenacker Useful for connecting a PR server to an upstream one - "test-package" checks whether the specified package version and arch is known in the database. - "test-pr" checks a specified output hash is found in the database. Otherwise it returns 'None' instead of a new value. - "max-package-pr" returns the highest PR number for (version, arch) entries in the database, and 0 if not found Add new DB functions supporting the above, plus test_value() which tells whether a given value is available for the specified package and architecture. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/client.py | 23 ++++++++++++++++++++- lib/prserv/db.py | 48 +++++++++++++++++++++++++++++++++++++++++++- lib/prserv/serv.py | 28 ++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 7bc5188c53..8471ee3046 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -20,6 +20,27 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): if response: return response["value"] + async def test_pr(self, version, pkgarch, checksum): + response = await self.invoke( + {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} + ) + if response: + return response["value"] + + async def test_package(self, version, pkgarch): + response = await self.invoke( + {"test-package": {"version": version, "pkgarch": pkgarch}} + ) + if response: + return response["value"] + + async def max_package_pr(self, version, pkgarch): + response = await self.invoke( + {"max-package-pr": {"version": version, "pkgarch": pkgarch}} + ) + if response: + return response["value"] + async def importone(self, version, pkgarch, checksum, value): response = await self.invoke( {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}} @@ -44,7 +65,7 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): class PRClient(bb.asyncrpc.Client): def __init__(self): super().__init__() - self._add_methods("getPR", "importone", "export", "is_readonly") + self._add_methods("getPR", "test_pr", "test_package", "importone", "export", "is_readonly") def _get_async_client(self): return PRAsyncClient() diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 7bc2b2dc2d..14b36ea6c9 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -78,12 +78,58 @@ class PRTable(object): self.sync() self.dirty = False - def _get_value_hist(self, version, pkgarch, checksum): + def test_package(self, version, pkgarch): + """Returns whether the specified package version is found in the database for the specified architecture""" + + # Just returns the value if found or None otherwise + data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, + (version, pkgarch)) + row=data.fetchone() + if row is not None: + return True + else: + return False + + def test_value(self, version, pkgarch, value): + """Returns whether the specified value is found in the database for the specified package and architecture""" + + # Just returns the value if found or None otherwise + data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, + (version, pkgarch, value)) + row=data.fetchone() + if row is not None: + return True + else: + return False + + def find_value(self, version, pkgarch, checksum): + """Unlike get_value, just returns the value if found or None otherwise. Doesn't create a new value""" + data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, (version, pkgarch, checksum)) row=data.fetchone() if row is not None: return row[0] + else: + return None + + def find_max_value(self, version, pkgarch): + """Returns the greatest value for (version, pkgarch), or "0" if not found. Doesn't create a new value""" + + data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), + (version, pkgarch)) + row = data.fetchone() + if row is not None: + return row[0] + else: + return "0" + + def _get_value_hist(self, version, pkgarch, checksum): + + val = find_value(self, version, pkgarch, checksum) + + if val is not None: + return val else: #no value found, try to insert if self.read_only: diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index efb2e0cf93..473dfa790c 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -26,6 +26,9 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): self.handlers.update({ "get-pr": self.handle_get_pr, + "test-pr": self.handle_test_pr, + "test-package": self.handle_test_package, + "max-package-pr": self.handle_max_package_pr, "import-one": self.handle_import_one, "export": self.handle_export, "is-readonly": self.handle_is_readonly, @@ -43,6 +46,31 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): else: self.server.table.sync_if_dirty() + async def handle_test_pr(self, request): + '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value''' + version = request["version"] + pkgarch = request["pkgarch"] + checksum = request["checksum"] + + value = self.server.table.find_value(version, pkgarch, checksum) + return {"value": value} + + async def handle_test_package(self, request): + '''Tells whether there are entries for (version, pkgarch) in the db. Returns True or False''' + version = request["version"] + pkgarch = request["pkgarch"] + + value = self.server.table.test_package(version, pkgarch) + return {"value": value} + + async def handle_max_package_pr(self, request): + '''Finds the greatest PR value for (version, pkgarch) in the db. Returns 0 if no entry was found''' + version = request["version"] + pkgarch = request["pkgarch"] + + value = self.server.table.find_max_value(version, pkgarch) + return {"value": value} + async def handle_get_pr(self, request): version = request["version"] pkgarch = request["pkgarch"] From patchwork Fri Mar 29 14:39:53 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41649 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id E4DD8CD1292 for ; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web11.32.1711723228349102969 for ; Fri, 29 Mar 2024 07:40:28 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=krBrEqof; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id E04501BF209; Fri, 29 Mar 2024 14:40:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723227; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=ewbCccP+tUGDul/E9eAtsKyOg+SD070Nt1YF32bCuK8=; b=krBrEqof2OIj03nZ9Xp6ZsNxKM1DSLqKZywTZavqRf0IdMOWt8LDEoCeKjtE0cZElgGScg w+/RZLUgMknCiXJb1ifoYoYGlYf8vEUQBTEEYUrRL1P3FAMq06Xl6o81rT27SWL6SvwVog 4riepnh2qDpYbEik7D6bzD3mE6yC4BzNBaBy9ZL7oxJ1uiM4x29TjaAxP3QwWQkHSUeA34 UYcvZ7JJPsaYx0iY121URKunT62ZClz3N8KLqn4dT6cEKpbq2iv46YesGYktvdbC4yYzzc Ge7jzq5EcI+ZUntK0VQb51j0KlYWy+K9Nu4CgQOCuSU5oJLjijJSdkqZRF922A== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 09/12] prserv: remove redundant exception handler Date: Fri, 29 Mar 2024 15:39:53 +0100 Message-Id: <20240329143956.1602707-10-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:28 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16030 From: Michael Opdenacker This exception handler is already present in db.py's get_value() code. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/serv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 473dfa790c..127036f9cc 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -82,8 +82,6 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): response = {"value": value} except prserv.NotFoundError: self.logger.error("can not find value for (%s, %s)",version, checksum) - except sqlite3.Error as exc: - self.logger.error(str(exc)) return response From patchwork Fri Mar 29 14:39:54 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41652 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id D8412CD11DD for ; Fri, 29 Mar 2024 14:40:38 +0000 (UTC) Received: from relay9-d.mail.gandi.net (relay9-d.mail.gandi.net [217.70.183.199]) by mx.groups.io with SMTP id smtpd.web10.25.1711723228741339574 for ; Fri, 29 Mar 2024 07:40:29 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=Sz5Zy9Mg; spf=pass (domain: bootlin.com, ip: 217.70.183.199, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 50BFEFF804; Fri, 29 Mar 2024 14:40:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723227; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=CVZfoH7FOKEoFhEtJi3hRHM9nBKmUs2eTXO670wRawI=; b=Sz5Zy9Mgv7JNqE6ftmffMG+cHIBZSIuNJV2fIRkqyFplb/adXc52XOtR8FS3g3xpkE64lT RZ2kbLsqp134NgQuVptC+hXUXtW+FftPSRGv+48AHFnopqroWpJlZIvJ/qGQ85thckF5RP qa4vU3eTFPeoa0q56RKA7gSOcduaqMn6BSnotpci9xVJxvD8pfbOR9XIN1F/ngC//r7QP3 DPJ0iwSm1Eg22BzTcNGiiFfDlZV4FlDx5o1lPJmkPxOnVOfwafHsYx58v4KwxcqZnnuwur sH46okgyz3iQ8YCa6DzcRhCv+EkxP6jcPGTH9xW21xqMEOYCK5XpUuSvONtZkA== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 10/12] prserv: correct error message Date: Fri, 29 Mar 2024 15:39:54 +0100 Message-Id: <20240329143956.1602707-11-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:38 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16032 From: Michael Opdenacker according to db.py, prserv.NotFoundError is returned here when adding a new value to the database failed Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/serv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 127036f9cc..604df6ce61 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -81,7 +81,7 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): value = self.server.table.get_value(version, pkgarch, checksum) response = {"value": value} except prserv.NotFoundError: - self.logger.error("can not find value for (%s, %s)",version, checksum) + self.logger.error("failure storing value in database for (%s, %s)",version, checksum) return response From patchwork Fri Mar 29 14:39:55 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41655 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id EB8F0CD1283 for ; Fri, 29 Mar 2024 14:40:38 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web11.34.1711723229161607166 for ; Fri, 29 Mar 2024 07:40:29 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=SmVEtKZX; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id AB5701BF208; Fri, 29 Mar 2024 14:40:27 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723227; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=uthPN0jwdMaTY5ZEVk3jHfzsliVnONZ7KSDMw0YcN6w=; b=SmVEtKZXZ0E63Hb6K1gkCrmSpI4gPhO1UcT4c9DmdvIwhQ1d6itrD8onASedbT2/eubp+b 0EW/knAC3p/FGgUg2kWM0RVi3Bo/ob0y6FtkODhL5D1AJKCJFhvddBo2Rsqg7k384XQu8R i6i/H2R0OhnUNtKXzqY5D6NzRHWQNMXoNwoeK8pG02hY1MbER1gdGRteVIxMx2rMpnlBar Yb6GJYtbIwDU2nU7JAn+yBBanFOaoyUqoiaPa0e+80ZNh4erwD1bMVM+g9TBTEmyoQ8yU3 MVR72INsWVhY2ed9Fky8X0kIr1KfzW7d6jQo1vR3v2W3p36H69UIymW8iJCVww== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 11/12] prserv: remove unnecessary code Date: Fri, 29 Mar 2024 15:39:55 +0100 Message-Id: <20240329143956.1602707-12-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:38 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16033 From: Michael Opdenacker In db.py, the ifnull() statement guarantees that the SQL request will return a value. It's therefore unnecessary to test the case when no value is found. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- lib/prserv/db.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/prserv/db.py b/lib/prserv/db.py index 14b36ea6c9..fddea923de 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -135,11 +135,7 @@ class PRTable(object): if self.read_only: data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) - row = data.fetchone() - if row is not None: - return row[0] - else: - return 0 + return data.fetchone()[0] try: self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" @@ -172,11 +168,7 @@ class PRTable(object): if self.read_only: data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), (version, pkgarch)) - row = data.fetchone() - if row is not None: - return row[0] - else: - return 0 + return data.fetchone()[0] try: self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" From patchwork Fri Mar 29 14:39:56 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Opdenacker X-Patchwork-Id: 41654 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 03D17CD1290 for ; Fri, 29 Mar 2024 14:40:39 +0000 (UTC) Received: from relay8-d.mail.gandi.net (relay8-d.mail.gandi.net [217.70.183.201]) by mx.groups.io with SMTP id smtpd.web10.26.1711723229560428464 for ; Fri, 29 Mar 2024 07:40:30 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@bootlin.com header.s=gm1 header.b=OOu3Ap28; spf=pass (domain: bootlin.com, ip: 217.70.183.201, mailfrom: michael.opdenacker@bootlin.com) Received: by mail.gandi.net (Postfix) with ESMTPSA id 103D01BF20A; Fri, 29 Mar 2024 14:40:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=bootlin.com; s=gm1; t=1711723228; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=3YjOPBd+p/2Vclb251ml6/lEjMRJpb8URJLtbdkm7X4=; b=OOu3Ap28eWq1svBD9kqhQQtbpkSd3Bikxlj0dHU/xnsNON4McmrzxF5+/KLqQureNSs6So 7NO0VfvBk9dMk1lTMxT7nDHutpXuu2UQTWhS1V/1O1lxA5USSKtGlxSLtIyyVxfcS4yW7P Ps8dvgilgPwTWQAiTfWZEkMIve7M9OJ4DvmxaDXwVVmb9GubOPuYbLbnSpJTUkvWicUrau voSNRbtKY8D7bvqg+rjZWV9jt84ha2Pof82mCHVHUNgjJ4s1blZffL4gZxe7hW7zEtLn4t 4UDfI6QSQsiWrduh2O4mX//cOk33lsdsFozUJHZsPcBzHwZJOUSkl6LFGNHRjQ== From: michael.opdenacker@bootlin.com To: bitbake-devel@lists.openembedded.org Cc: Michael Opdenacker , Joshua Watt , Tim Orling Subject: [PATCH 12/12] prserv: add "upstream" server support Date: Fri, 29 Mar 2024 15:39:56 +0100 Message-Id: <20240329143956.1602707-13-michael.opdenacker@bootlin.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> References: <20240329143956.1602707-1-michael.opdenacker@bootlin.com> MIME-Version: 1.0 X-GND-Sasl: michael.opdenacker@bootlin.com List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Fri, 29 Mar 2024 14:40:39 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/16034 From: Michael Opdenacker Introduce a PRSERVER_UPSTREAM variable that makes the local PR server connect to an "upstream" one. This makes it possible to implement local fixes to an upstream package (revision "x", in a way that gives the local update priority (revision "x.y"). Set the comments in the handle_get_pr() function in serv.py for details about the calculation of the local revision. Signed-off-by: Michael Opdenacker Cc: Joshua Watt Cc: Tim Orling --- bin/bitbake-prserv | 15 +++++- lib/prserv/__init__.py | 15 ++++++ lib/prserv/client.py | 1 + lib/prserv/db.py | 30 ++++++++++++ lib/prserv/serv.py | 106 ++++++++++++++++++++++++++++++++++++----- 5 files changed, 154 insertions(+), 13 deletions(-) diff --git a/bin/bitbake-prserv b/bin/bitbake-prserv index ad0a069401..e39d0fba87 100755 --- a/bin/bitbake-prserv +++ b/bin/bitbake-prserv @@ -70,12 +70,25 @@ def main(): action="store_true", help="open database in read-only mode", ) + parser.add_argument( + "-u", + "--upstream", + default=os.environ.get("PRSERVER_UPSTREAM", None), + help="Upstream PR service (host:port)", + ) args = parser.parse_args() prserv.init_logger(os.path.abspath(args.log), args.loglevel) if args.start: - ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only) + ret=prserv.serv.start_daemon( + args.file, + args.host, + args.port, + os.path.abspath(args.log), + args.read_only, + args.upstream + ) elif args.stop: ret=prserv.serv.stop_daemon(args.host, args.port) else: diff --git a/lib/prserv/__init__.py b/lib/prserv/__init__.py index 0e0aa34d0e..2ee6a28c04 100644 --- a/lib/prserv/__init__.py +++ b/lib/prserv/__init__.py @@ -8,6 +8,7 @@ __version__ = "1.0.0" import os, time import sys, logging +from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS def init_logger(logfile, loglevel): numeric_level = getattr(logging, loglevel.upper(), None) @@ -18,3 +19,17 @@ def init_logger(logfile, loglevel): class NotFoundError(Exception): pass + +async def create_async_client(addr): + from . import client + + c = client.PRAsyncClient() + + try: + (typ, a) = parse_address(addr) + await c.connect_tcp(*a) + return c + + except Exception as e: + await c.close() + raise e diff --git a/lib/prserv/client.py b/lib/prserv/client.py index 8471ee3046..89760b6f74 100644 --- a/lib/prserv/client.py +++ b/lib/prserv/client.py @@ -6,6 +6,7 @@ import logging import bb.asyncrpc +from . import create_async_client logger = logging.getLogger("BitBake.PRserv") diff --git a/lib/prserv/db.py b/lib/prserv/db.py index fddea923de..b581bbf072 100644 --- a/lib/prserv/db.py +++ b/lib/prserv/db.py @@ -124,6 +124,36 @@ class PRTable(object): else: return "0" + def find_max_value(self, version, pkgarch): + """Returns the greatest value for (version, pkgarch), or "0" if not found. Doesn't create a new value""" + + data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), + (version, pkgarch)) + row = data.fetchone() + if row is not None: + return row[0] + else: + return "0" + + def find_new_subvalue(self, version, pkgarch, base): + """Returns the greatest ".y" value for (version, pkgarch), or ".1" if not found. Doesn't store a new value""" + # The code doesn't propose ".0" because it would store it as "" was declared as an integer + + data = self._execute("SELECT ifnull(max(value)+0.1, %s.1) FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (base, self.table, base), + (version, pkgarch)) + return data.fetchone()[0] + + def store_value(self, version, pkgarch, checksum, value): + '''Store new value in the database''' + + try: + self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), + (version, pkgarch, checksum, value)) + except sqlite3.IntegrityError as exc: + logger.error(str(exc)) + + self.dirty = True + def _get_value_hist(self, version, pkgarch, checksum): val = find_value(self, version, pkgarch, checksum) diff --git a/lib/prserv/serv.py b/lib/prserv/serv.py index 604df6ce61..1461e27ad2 100644 --- a/lib/prserv/serv.py +++ b/lib/prserv/serv.py @@ -12,6 +12,7 @@ import sqlite3 import prserv import prserv.db import errno +from . import create_async_client import bb.asyncrpc logger = logging.getLogger("BitBake.PRserv") @@ -77,13 +78,86 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): checksum = request["checksum"] response = None - try: - value = self.server.table.get_value(version, pkgarch, checksum) - response = {"value": value} - except prserv.NotFoundError: - self.logger.error("failure storing value in database for (%s, %s)",version, checksum) - return response + if self.upstream_client is None: + try: + value = self.server.table.get_value(version, pkgarch, checksum) + response = {"value": value} + except prserv.NotFoundError: + self.logger.error("failure storing value in database for (%s, %s)", version, checksum) + + return response + + # We have an upstream server. + # Check whether the local server already knows the requested configuration + # Here we use find_value(), not get_value(), because we don't want + # to unconditionally add a new generated value to the database. If the configuration + # is a new one, the generated value we will add will depend on what's on the upstream server. + + value = self.server.table.find_value(version, pkgarch, checksum) + + if value is not None: + + # The configuration is already known locally. Let's use it. + + return {"value": value} + + # The configuration is a new one for the local server + # Let's ask the upstream server whether it knows it + + known_upstream = await self.upstream_client.test_package(version, pkgarch) + + if not known_upstream: + + # The package is not known upstream, must be a local-only package + # Let's compute the PR number using the local-only method + + try: + value = self.server.table.get_value(version, pkgarch, checksum) + response = {"value": value} + except prserv.NotFoundError: + self.logger.error("failure storing value in database for (%s, %s)", version, checksum) + + return response + + # The package is known upstream, let's ask the upstream server + # whether it knows our new output hash + + value = await self.upstream_client.test_pr(version, pkgarch, checksum) + + if value is not None: + + # Upstream knows this output hash, let's store it and use it too. + + if not self.server.read_only: + self.server.table.store_value(version, pkgarch, checksum, value) + # If the local server is read only, won't be able to store the new + # value in the database and will have to keep asking the upstream server + + return {"value": value} + + # The output hash doesn't exist upstream, get the most recent number from upstream (x) + # Then, we want to have a new PR value for the local server: x.y + + upstream_max_value = await self.upstream_client.max_package_pr(version, pkgarch) + subvalue = self.server.table.find_new_subvalue(version, pkgarch, upstream_max_value) + + if not self.server.read_only: + self.server.table.store_value(version, pkgarch, checksum, subvalue) + + return {"value": subvalue} + + async def process_requests(self): + if self.server.upstream is not None: + self.upstream_client = await create_async_client(self.server.upstream) + else: + self.upstream_client = None + + try: + await super().process_requests() + finally: + if self.upstream_client is not None: + await self.upstream_client.close() async def handle_import_one(self, request): response = None @@ -117,11 +191,12 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): return {"readonly": self.server.read_only} class PRServer(bb.asyncrpc.AsyncServer): - def __init__(self, dbfile, read_only=False): + def __init__(self, dbfile, read_only=False, upstream=None): super().__init__(logger) self.dbfile = dbfile self.table = None self.read_only = read_only + self.upstream = upstream def accept_client(self, socket): return PRServerClient(socket, self) @@ -134,6 +209,9 @@ class PRServer(bb.asyncrpc.AsyncServer): self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % (self.dbfile, self.address, str(os.getpid()))) + if self.upstream is not None: + self.logger.info("And upstream PRServer: %s " % (self.upstream)) + return tasks async def stop(self): @@ -147,14 +225,15 @@ class PRServer(bb.asyncrpc.AsyncServer): self.table.sync() class PRServSingleton(object): - def __init__(self, dbfile, logfile, host, port): + def __init__(self, dbfile, logfile, host, port, upstream): self.dbfile = dbfile self.logfile = logfile self.host = host self.port = port + self.upstream = upstream def start(self): - self.prserv = PRServer(self.dbfile) + self.prserv = PRServer(self.dbfile, upstream=self.upstream) self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port) self.process = self.prserv.serve_as_process(log_level=logging.WARNING) @@ -233,7 +312,7 @@ def run_as_daemon(func, pidfile, logfile): os.remove(pidfile) os._exit(0) -def start_daemon(dbfile, host, port, logfile, read_only=False): +def start_daemon(dbfile, host, port, logfile, read_only=False, upstream=None): ip = socket.gethostbyname(host) pidfile = PIDPREFIX % (ip, port) try: @@ -249,7 +328,7 @@ def start_daemon(dbfile, host, port, logfile, read_only=False): dbfile = os.path.abspath(dbfile) def daemon_main(): - server = PRServer(dbfile, read_only=read_only) + server = PRServer(dbfile, read_only=read_only, upstream=upstream) server.start_tcp_server(ip, port) server.serve_forever() @@ -336,6 +415,9 @@ def auto_start(d): host = host_params[0].strip().lower() port = int(host_params[1]) + + upstream = d.getVar("PRSERV_UPSTREAM") or None + if is_local_special(host, port): import bb.utils cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE")) @@ -350,7 +432,7 @@ def auto_start(d): auto_shutdown() if not singleton: bb.utils.mkdirhier(cachedir) - singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port) + singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port, upstream) singleton.start() if singleton: host = singleton.host