From patchwork Thu Sep 28 17:05:47 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joshua Watt X-Patchwork-Id: 31320 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 751CBE732F8 for ; Thu, 28 Sep 2023 17:05:59 +0000 (UTC) Received: from mail-oi1-f174.google.com (mail-oi1-f174.google.com [209.85.167.174]) by mx.groups.io with SMTP id smtpd.web11.18860.1695920758795780890 for ; Thu, 28 Sep 2023 10:05:58 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=Ctqul+bG; spf=pass (domain: gmail.com, ip: 209.85.167.174, mailfrom: jpewhacker@gmail.com) Received: by mail-oi1-f174.google.com with SMTP id 5614622812f47-3af5b5d7f16so1619161b6e.0 for ; Thu, 28 Sep 2023 10:05:58 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1695920757; x=1696525557; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=bw9taJarT/airkFdi9rJBJ9ECeJqhRQXfeJolDXHxAo=; b=Ctqul+bGWqHTwUQ0J+JCRVH/ypT2m6fmLi85dQTgZi+doh5fgrfqwPMfdxui6FLQLr xkRpljwAtEiXPy/Bz1oEwdSDs7dylDVCwkN9vYS5h2uKA1zukDIX0hQD35Wpj9XQaDPX eRHWkqE7QtlnLDaDZiluh1vArTMmCH63yO3jtV53oj0anWfdljd9aGarCMTGhdDGwTV3 WxGnVuDR+wB/6JnApdSwIZpJ9n8BqADJM+eaS/L9PbS6AFJfpY2QWqbPBywsBXazhqsg RzsJWe1TF+KIvQC5PWmImzPLBEUYQoKgUs+Fku2+cKAouNUxAGYIoqqSFrPckH05nwX9 aEbA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1695920757; x=1696525557; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=bw9taJarT/airkFdi9rJBJ9ECeJqhRQXfeJolDXHxAo=; b=e7djmMxjlM0HTdUJfTpikcZCHll3N5CDvV5nt1ogdc1XRLsL2wccLam/GhJoPxWCJ6 ZyF2VODhP4HzpHT1rfVwC4FKCDynNMz8S5esYl+SHy1nBH+VCk7PEQs4OOzlUVeNSRr/ G47p85r2Ll1T5fKf/0ST67Re1FZV/Wb8N0yjQv6UdEidYD81baS6HblBHIjmEDL2vPTn DoaqWYyLWpj3Z+/5Ji5BJw2bUjixJWK+LhujnXotsv2O1CW4D3kfnX5qLmOe8W3LZN3O 5SUB9OfMQO9+N8WOcCgrKvofZQEjxwG9yDQWjJa3LdNxOvr6am6NkQ28yvZZoo3B7MWr ysCw== X-Gm-Message-State: AOJu0Yy1nIW9aGjK1n3f9SwUB3eOSPr9k5wDfX4e4jPsdZ/CcUg9r2zZ V9WpZ/EvC+E+an9yt9W6oFgPg3bxh1A= X-Google-Smtp-Source: AGHT+IEfD7t+uYlVg8NPJeES+xu/P+pMcHiqGcveapqpHK+p/ZE3LL2wNt8kh/MNYoaxW+oI7awnCw== X-Received: by 2002:a05:6870:148c:b0:1dc:bb65:f2be with SMTP id k12-20020a056870148c00b001dcbb65f2bemr1893607oab.10.1695920757259; Thu, 28 Sep 2023 10:05:57 -0700 (PDT) Received: from localhost.localdomain ([2601:282:4300:19e0::f27c]) by smtp.gmail.com with ESMTPSA id h3-20020a056870860300b001dd533c4bd1sm1389772oal.1.2023.09.28.10.05.56 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 28 Sep 2023 10:05:56 -0700 (PDT) From: Joshua Watt X-Google-Original-From: Joshua Watt To: bitbake-devel@lists.openembedded.org Cc: Joshua Watt Subject: [bitbake-devel][RFC 1/5] asyncrpc: Abstract client socket Date: Thu, 28 Sep 2023 11:05:47 -0600 Message-Id: <20230928170551.4193224-2-JPEWhacker@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230928170551.4193224-1-JPEWhacker@gmail.com> References: <20230928170551.4193224-1-JPEWhacker@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 28 Sep 2023 17:05:59 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/15143 Rewrites the asyncrpc client code to make it possible to have other transport backends that are not stream based (e.g. websockets which are message based). Signed-off-by: Joshua Watt --- bitbake/lib/bb/asyncrpc/client.py | 135 ++++++++++++++++++------------ bitbake/lib/hashserv/client.py | 24 +++--- bitbake/lib/prserv/client.py | 8 +- 3 files changed, 95 insertions(+), 72 deletions(-) diff --git a/bitbake/lib/bb/asyncrpc/client.py b/bitbake/lib/bb/asyncrpc/client.py index fa042bbe87c..335da09d8c6 100644 --- a/bitbake/lib/bb/asyncrpc/client.py +++ b/bitbake/lib/bb/asyncrpc/client.py @@ -13,10 +13,74 @@ import sys from . import chunkify, DEFAULT_MAX_CHUNK +class StreamParser(object): + def __init__(self, reader, writer, timeout, max_chunk): + self.reader = reader + self.writer = writer + self.timeout = timeout + self.max_chunk = max_chunk + + async def setup_connection(self, proto_name, proto_version): + s = "%s %s\n\n" % (proto_name, proto_version) + self.writer.write(s.encode("utf-8")) + await self.writer.drain() + + async def invoke(self, msg): + async def get_line(): + try: + line = await asyncio.wait_for(self.reader.readline(), self.timeout) + except asyncio.TimeoutError: + raise ConnectionError("Timed out waiting for server") + + if not line: + raise ConnectionError("Connection closed") + + line = line.decode("utf-8") + + if not line.endswith("\n"): + raise ConnectionError("Bad message %r" % (line)) + + return line + + for c in chunkify(json.dumps(msg), self.max_chunk): + self.writer.write(c.encode("utf-8")) + await self.writer.drain() + + l = await get_line() + + m = json.loads(l) + if m and "chunk-stream" in m: + lines = [] + while True: + l = (await get_line()).rstrip("\n") + if not l: + break + lines.append(l) + + m = json.loads("".join(lines)) + + return m + + async def send(self, msg): + self.writer.write(("%s\n" % msg).encode("utf-8")) + await self.writer.drain() + + async def recv(self): + l = await self.reader.readline() + if not l: + raise ConnectionError("Connection closed") + return l.decode("utf-8").rstrip() + + async def close(self): + self.reader = None + if self.writer is not None: + self.writer.close() + self.writer = None + + class AsyncClient(object): def __init__(self, proto_name, proto_version, logger, timeout=30): - self.reader = None - self.writer = None + self.socket = None self.max_chunk = DEFAULT_MAX_CHUNK self.proto_name = proto_name self.proto_version = proto_version @@ -25,7 +89,8 @@ class AsyncClient(object): async def connect_tcp(self, address, port): async def connect_sock(): - return await asyncio.open_connection(address, port) + reader, writer = await asyncio.open_connection(address, port) + return StreamParser(reader, writer, self.timeout, self.max_chunk) self._connect_sock = connect_sock @@ -40,27 +105,24 @@ class AsyncClient(object): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) sock.connect(os.path.basename(path)) finally: - os.chdir(cwd) - return await asyncio.open_unix_connection(sock=sock) + os.chdir(cwd) + reader, writer = await asyncio.open_unix_connection(sock=sock) + return StreamParser(reader, writer, self.timeout, self.max_chunk) self._connect_sock = connect_sock async def setup_connection(self): - s = '%s %s\n\n' % (self.proto_name, self.proto_version) - self.writer.write(s.encode("utf-8")) - await self.writer.drain() + await self.socket.setup_connection(self.proto_name, self.proto_version) async def connect(self): - if self.reader is None or self.writer is None: - (self.reader, self.writer) = await self._connect_sock() + if self.socket is None: + self.socket = await self._connect_sock() await self.setup_connection() async def close(self): - self.reader = None - - if self.writer is not None: - self.writer.close() - self.writer = None + if self.socket is not None: + await self.socket.close() + self.socket = None async def _send_wrapper(self, proc): count = 0 @@ -82,49 +144,14 @@ class AsyncClient(object): await self.close() count += 1 - async def send_message(self, msg): - async def get_line(): - try: - line = await asyncio.wait_for(self.reader.readline(), self.timeout) - except asyncio.TimeoutError: - raise ConnectionError("Timed out waiting for server") - - if not line: - raise ConnectionError("Connection closed") - - line = line.decode("utf-8") - - if not line.endswith("\n"): - raise ConnectionError("Bad message %r" % (line)) - - return line - + async def invoke(self, msg): async def proc(): - for c in chunkify(json.dumps(msg), self.max_chunk): - self.writer.write(c.encode("utf-8")) - await self.writer.drain() - - l = await get_line() - - m = json.loads(l) - if m and "chunk-stream" in m: - lines = [] - while True: - l = (await get_line()).rstrip("\n") - if not l: - break - lines.append(l) - - m = json.loads("".join(lines)) - - return m + return await self.socket.invoke(msg) return await self._send_wrapper(proc) async def ping(self): - return await self.send_message( - {'ping': {}} - ) + return await self.send_message({"ping": {}}) class Client(object): @@ -142,7 +169,7 @@ class Client(object): # required (but harmless) with it. asyncio.set_event_loop(self.loop) - self._add_methods('connect_tcp', 'ping') + self._add_methods("connect_tcp", "ping") @abc.abstractmethod def _get_async_client(self): diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py index b2aa1026ac9..2a3c1b662b6 100644 --- a/bitbake/lib/hashserv/client.py +++ b/bitbake/lib/hashserv/client.py @@ -28,12 +28,8 @@ class AsyncClient(bb.asyncrpc.AsyncClient): async def send_stream(self, msg): async def proc(): - self.writer.write(("%s\n" % msg).encode("utf-8")) - await self.writer.drain() - l = await self.reader.readline() - if not l: - raise ConnectionError("Connection closed") - return l.decode("utf-8").rstrip() + await self.socket.send(msg) + return await self.socket.recv() return await self._send_wrapper(proc) @@ -43,7 +39,7 @@ class AsyncClient(bb.asyncrpc.AsyncClient): if r != "ok": raise ConnectionError("Bad response from server %r" % r) elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL: - r = await self.send_message({"get-stream": None}) + r = await self.invoke({"get-stream": None}) if r != "ok": raise ConnectionError("Bad response from server %r" % r) elif new_mode != self.mode: @@ -67,7 +63,7 @@ class AsyncClient(bb.asyncrpc.AsyncClient): m["method"] = method m["outhash"] = outhash m["unihash"] = unihash - return await self.send_message({"report": m}) + return await self.invoke({"report": m}) async def report_unihash_equiv(self, taskhash, method, unihash, extra={}): await self._set_mode(self.MODE_NORMAL) @@ -75,31 +71,31 @@ class AsyncClient(bb.asyncrpc.AsyncClient): m["taskhash"] = taskhash m["method"] = method m["unihash"] = unihash - return await self.send_message({"report-equiv": m}) + return await self.invoke({"report-equiv": m}) async def get_taskhash(self, method, taskhash, all_properties=False): await self._set_mode(self.MODE_NORMAL) - return await self.send_message( + return await self.invoke( {"get": {"taskhash": taskhash, "method": method, "all": all_properties}} ) async def get_outhash(self, method, outhash, taskhash): await self._set_mode(self.MODE_NORMAL) - return await self.send_message( + return await self.invoke( {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method}} ) async def get_stats(self): await self._set_mode(self.MODE_NORMAL) - return await self.send_message({"get-stats": None}) + return await self.invoke({"get-stats": None}) async def reset_stats(self): await self._set_mode(self.MODE_NORMAL) - return await self.send_message({"reset-stats": None}) + return await self.invoke({"reset-stats": None}) async def backfill_wait(self): await self._set_mode(self.MODE_NORMAL) - return (await self.send_message({"backfill-wait": None}))["tasks"] + return (await self.invoke({"backfill-wait": None}))["tasks"] class Client(bb.asyncrpc.Client): diff --git a/bitbake/lib/prserv/client.py b/bitbake/lib/prserv/client.py index 69ab7a4ac9d..6b81356fac5 100644 --- a/bitbake/lib/prserv/client.py +++ b/bitbake/lib/prserv/client.py @@ -14,28 +14,28 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): super().__init__('PRSERVICE', '1.0', logger) async def getPR(self, version, pkgarch, checksum): - response = await self.send_message( + response = await self.invoke( {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}} ) if response: return response['value'] async def importone(self, version, pkgarch, checksum, value): - response = await self.send_message( + response = await self.invoke( {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}} ) if response: return response['value'] async def export(self, version, pkgarch, checksum, colinfo): - response = await self.send_message( + response = await self.invoke( {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}} ) if response: return (response['metainfo'], response['datainfo']) async def is_readonly(self): - response = await self.send_message( + response = await self.invoke( {'is-readonly': {}} ) if response: From patchwork Thu Sep 28 17:05:48 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joshua Watt X-Patchwork-Id: 31322 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 553FCE732F4 for ; Thu, 28 Sep 2023 17:06:09 +0000 (UTC) Received: from mail-ot1-f44.google.com (mail-ot1-f44.google.com [209.85.210.44]) by mx.groups.io with SMTP id smtpd.web11.18861.1695920760363124549 for ; Thu, 28 Sep 2023 10:06:00 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=Mmx575o+; spf=pass (domain: gmail.com, ip: 209.85.210.44, mailfrom: jpewhacker@gmail.com) Received: by mail-ot1-f44.google.com with SMTP id 46e09a7af769-6c6368a1a97so207809a34.1 for ; Thu, 28 Sep 2023 10:06:00 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1695920759; x=1696525559; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=mniGGYwxo14cEdvMYuiYDZFic/qIn7vlBumgwms2QPY=; b=Mmx575o+nH3jPTW1TvZfQqC1Y4VQwoFritgXDdX3nQPo8Q7Vlilsbqt37PVkjF5Jlt KV5XjIJMdzOx1u3fVPLHtkLWusPCzYhTtLzDhX6MnRHuli2nvpv1bhUP3E+z9/qlYF18 gVK0gagpitIYE8N8qRi78pAHrDnhVq0uk16qwbZ1BeosfCCWOF8w9ekDFLBQ47K/du6x +1/Da0yMOdHcDakPRA2c2UDl/krgyY9U4lEvwk8ufiPDhccm8S7mmttsDsbPQJawDD8G gEbrsNMyt7Se/qjvXZPG1gEpr6ITBVFLoQnfCWZlXAl8LIJeOPr+ZnITbPBWCcOteZpd +98A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1695920759; x=1696525559; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=mniGGYwxo14cEdvMYuiYDZFic/qIn7vlBumgwms2QPY=; b=XenBk+lEzfxSNB1IoVmaIR3pqI3r/SxDHAyd4TMDobmzkmMTEmwdoSfYcOssYbEMV3 tAkTGmshQmt4ItjJVRgMet1JtnX0EEw/IKxLraEblKWnJzJTTLQUfpMP9lWgzxLu+xWe tz31teAtVUwplsICgQ5DwGArENf6n/ftf8kqJVDTiQdGxoLJf2DrNBKpbev3CSwKwaLV TYHRJZJDdO9bgFoz/X3lySxDNchJAa7Ux/CO78BvFFx+GkOshhhXAuorZWpypPdVglsH gZYckpvjQdcPtL9Hi8xm+Onubf3HJ0bsDCFsik9xManEbhxWJEFAvGFUY3bxmiGJpkKQ o90Q== X-Gm-Message-State: AOJu0YyNQFzXvDBx16BoZ6WV6ZTyqF4oLxmyl5E7WzhZnfhjBcmBwmFo dP3Cohp9Oe2xI9iZwUcH1xtm+Orig2w= X-Google-Smtp-Source: AGHT+IE6nGwy/GvBhyvP4r7kyl4rQXTC0k35j9phIplPcVYYJXq/mWGlCbRKSfTdSGcRebI1bfKpuQ== X-Received: by 2002:a05:6870:e40e:b0:1b0:3f7f:673a with SMTP id n14-20020a056870e40e00b001b03f7f673amr1302319oag.25.1695920758903; Thu, 28 Sep 2023 10:05:58 -0700 (PDT) Received: from localhost.localdomain ([2601:282:4300:19e0::f27c]) by smtp.gmail.com with ESMTPSA id h3-20020a056870860300b001dd533c4bd1sm1389772oal.1.2023.09.28.10.05.57 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 28 Sep 2023 10:05:57 -0700 (PDT) From: Joshua Watt X-Google-Original-From: Joshua Watt To: bitbake-devel@lists.openembedded.org Cc: Joshua Watt Subject: [bitbake-devel][RFC 2/5] hashserv: Add remove API Date: Thu, 28 Sep 2023 11:05:48 -0600 Message-Id: <20230928170551.4193224-3-JPEWhacker@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230928170551.4193224-1-JPEWhacker@gmail.com> References: <20230928170551.4193224-1-JPEWhacker@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 28 Sep 2023 17:06:09 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/15144 Adds a `remove` API to the client and server that can be used to remove hash equivalence entries that match a particular critera Signed-off-by: Joshua Watt --- bitbake/lib/hashserv/client.py | 5 +++++ bitbake/lib/hashserv/server.py | 25 +++++++++++++++++++++++++ bitbake/lib/hashserv/tests.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py index 2a3c1b662b6..7d2b9cb394f 100644 --- a/bitbake/lib/hashserv/client.py +++ b/bitbake/lib/hashserv/client.py @@ -97,6 +97,10 @@ class AsyncClient(bb.asyncrpc.AsyncClient): await self._set_mode(self.MODE_NORMAL) return (await self.invoke({"backfill-wait": None}))["tasks"] + async def remove(self, where): + await self._set_mode(self.MODE_NORMAL) + return await self.invoke({"remove": {"where": where}}) + class Client(bb.asyncrpc.Client): def __init__(self): @@ -111,6 +115,7 @@ class Client(bb.asyncrpc.Client): "get_stats", "reset_stats", "backfill_wait", + "remove", ) def _get_async_client(self): diff --git a/bitbake/lib/hashserv/server.py b/bitbake/lib/hashserv/server.py index d40a2ab8f88..7e8aeefef30 100644 --- a/bitbake/lib/hashserv/server.py +++ b/bitbake/lib/hashserv/server.py @@ -186,6 +186,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): 'report-equiv': self.handle_equivreport, 'reset-stats': self.handle_reset_stats, 'backfill-wait': self.handle_backfill_wait, + 'remove': self.handle_remove, }) def validate_proto_version(self): @@ -499,6 +500,30 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): await self.backfill_queue.join() self.write_message(d) + async def handle_remove(self, request): + condition = request["where"] + if not isinstance(condition, dict): + raise TypeError("Bad condition type %s" % type(condition)) + + def do_remove(columns, table_name, cursor): + nonlocal condition + where = {} + for c in columns: + if c in condition and condition[c] is not None: + where[c] = condition[c] + + if where: + query = ('DELETE FROM %s WHERE ' % table_name) + ' AND '.join("%s=:%s" % (k, k) for k in where.keys()) + print(query) + cursor.execute(query, where) + + with closing(self.db.cursor()) as cursor: + do_remove(OUTHASH_TABLE_COLUMNS, "outhashes_v2", cursor) + do_remove(UNIHASH_TABLE_COLUMNS, "unihashes_v2", cursor) + self.db.commit() + + self.write_message({}) + def query_equivalent(self, cursor, method, taskhash): # This is part of the inner loop and must be as fast as possible cursor.execute( diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index f6b85aed85a..5f1ad585072 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py @@ -84,6 +84,7 @@ class HashEquivalenceCommonTests(object): result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') + return taskhash, outhash, unihash def test_create_equivalent(self): # Tests that a second reported task with the same outhash will be @@ -125,6 +126,34 @@ class HashEquivalenceCommonTests(object): self.assertClientGetHash(self.client, taskhash, unihash) + def test_remove_taskhash(self): + taskhash, outhash, unihash = self.test_create_hash() + self.client.remove({"taskhash": taskhash}) + self.assertClientGetHash(self.client, taskhash, None) + + result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash) + self.assertIsNone(result_outhash) + + def test_remove_unihash(self): + taskhash, outhash, unihash = self.test_create_hash() + self.client.remove({"unihash": unihash}) + self.assertClientGetHash(self.client, taskhash, None) + + def test_remove_outhash(self): + taskhash, outhash, unihash = self.test_create_hash() + self.client.remove({"outhash": outhash}) + + result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash) + self.assertIsNone(result_outhash) + + def test_remove_method(self): + taskhash, outhash, unihash = self.test_create_hash() + self.client.remove({"method": self.METHOD}) + self.assertClientGetHash(self.client, taskhash, None) + + result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash) + self.assertIsNone(result_outhash) + def test_huge_message(self): # Simple test that hashes can be created taskhash = 'c665584ee6817aa99edfc77a44dd853828279370' From patchwork Thu Sep 28 17:05:49 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joshua Watt X-Patchwork-Id: 31321 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 55121E732F9 for ; Thu, 28 Sep 2023 17:06:09 +0000 (UTC) Received: from mail-oo1-f41.google.com (mail-oo1-f41.google.com [209.85.161.41]) by mx.groups.io with SMTP id smtpd.web10.19054.1695920762078512614 for ; Thu, 28 Sep 2023 10:06:02 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=SF9a/P8K; spf=pass (domain: gmail.com, ip: 209.85.161.41, mailfrom: jpewhacker@gmail.com) Received: by mail-oo1-f41.google.com with SMTP id 006d021491bc7-57bc2c2f13dso4123819eaf.2 for ; Thu, 28 Sep 2023 10:06:01 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1695920760; x=1696525560; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=opxTwZ1am1DanXDb688FjBHsuGD2K0vtpsvToRLtLr8=; b=SF9a/P8KHeoA3of4SDmZFKUAXEda43mECR1kpuCCcnNlkFWRJInCSsnS0A3vWSR2qu 5okNJFatKfIx4j2to21szxKkVIL5LMZZzHHSSGkZfaqh/PlA5sSIHiMhHDca3Nyw07Ul j2LuaPtCWZ236tTSYNdIeiATU3aLEgkneKSh6Z0cuE3MQ7h8UhHNZ5GJ+0K1n2cDv4vS ajXy7oQ3pa/GeEoa5miZVj3l1OQos6GlRhkKdJbNe1oU0Gpzu2L8WI7itJ00KOSW8h8q UneRtTDvVQ+70+AvHXqTwU6hoz3LMdRLQsFy6qMxZGgNKKN01zwDAtGKpCXfYapsz7I3 fseQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1695920760; x=1696525560; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=opxTwZ1am1DanXDb688FjBHsuGD2K0vtpsvToRLtLr8=; b=QaFysnBZxSs2hAb/AxOo1vsi4JmAyQdo4nrvzIpjWFDUfTQnCjQkK1u5b8Z3mEFW2B Uryj4nnoZ+35BTPxs0XMooqkm+SOk0cgh0xASO+cQSnYI32Ni3BrWUdUFSG3oDX0HhTh YQPA0wyqf4VPM2lPjB0JWz2ZmfVcKQ25lkM+AXZTOTxWTGFyKMh8VkjMGix1vkAadQME SZb6XasZsWuMymLiH85YD6wECHe1RqFTbEv6KMk7KelF0c/vd0QDHfWiIU9Be3H6QhpB /MBTf9gJYpe8+qJ8z8MCqqrXsa48pu0gAdk/1ZriKVCxogbQ4QwOXzZt54QVUMdW6C9U kclw== X-Gm-Message-State: AOJu0YyNqkBhxR4fZqgiVVK8vreQ9w4DZRJCAntXaoDBL9fRmLc3y8vn ReOoA5J0QWl1QKDKYKs621PF+zgphs4= X-Google-Smtp-Source: AGHT+IGi49CCN9TPeV0ZSqXHERtNdze1ltFwiTHXMTP7T8vhNpwaLXB9vxNhWMSoNNB/DC7zLo3gFA== X-Received: by 2002:a05:6870:d60e:b0:1dc:ddd0:6ee1 with SMTP id a14-20020a056870d60e00b001dcddd06ee1mr1866196oaq.38.1695920760560; Thu, 28 Sep 2023 10:06:00 -0700 (PDT) Received: from localhost.localdomain ([2601:282:4300:19e0::f27c]) by smtp.gmail.com with ESMTPSA id h3-20020a056870860300b001dd533c4bd1sm1389772oal.1.2023.09.28.10.05.59 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 28 Sep 2023 10:05:59 -0700 (PDT) From: Joshua Watt X-Google-Original-From: Joshua Watt To: bitbake-devel@lists.openembedded.org Cc: Joshua Watt Subject: [bitbake-devel][RFC 3/5] bitbake-hashclient: Add remove subcommand Date: Thu, 28 Sep 2023 11:05:49 -0600 Message-Id: <20230928170551.4193224-4-JPEWhacker@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230928170551.4193224-1-JPEWhacker@gmail.com> References: <20230928170551.4193224-1-JPEWhacker@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 28 Sep 2023 17:06:09 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/15145 Adds a subcommand to invoke the remove API on the server [YOCTO #15064] Signed-off-by: Joshua Watt --- bitbake/bin/bitbake-hashclient | 13 +++++++++++++ bitbake/lib/hashserv/server.py | 10 +++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bitbake/bin/bitbake-hashclient b/bitbake/bin/bitbake-hashclient index 494f17592ac..d09104336ab 100755 --- a/bitbake/bin/bitbake-hashclient +++ b/bitbake/bin/bitbake-hashclient @@ -113,6 +113,14 @@ def main(): with lock: pbar.update() + def handle_remove(args, client): + where = {k: v for k, v in args.where} + if where: + result = client.remove(where) + print("Removed %d row(s)" % (result["count"])) + else: + print("No query specified") + parser = argparse.ArgumentParser(description='Hash Equivalence Client') parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")') parser.add_argument('--log', default='WARNING', help='Set logging level') @@ -137,6 +145,11 @@ def main(): help='Include string in outhash') stress_parser.set_defaults(func=handle_stress) + remove_parser = subparsers.add_parser('remove', help="Remove hash entries") + remove_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[], + help="Remove entries from table where KEY == VALUE") + remove_parser.set_defaults(func=handle_remove) + args = parser.parse_args() logger = logging.getLogger('hashserv') diff --git a/bitbake/lib/hashserv/server.py b/bitbake/lib/hashserv/server.py index 7e8aeefef30..99ba904b1f0 100644 --- a/bitbake/lib/hashserv/server.py +++ b/bitbake/lib/hashserv/server.py @@ -516,13 +516,17 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): query = ('DELETE FROM %s WHERE ' % table_name) + ' AND '.join("%s=:%s" % (k, k) for k in where.keys()) print(query) cursor.execute(query, where) + return cursor.rowcount + return 0 + + count = 0 with closing(self.db.cursor()) as cursor: - do_remove(OUTHASH_TABLE_COLUMNS, "outhashes_v2", cursor) - do_remove(UNIHASH_TABLE_COLUMNS, "unihashes_v2", cursor) + count += do_remove(OUTHASH_TABLE_COLUMNS, "outhashes_v2", cursor) + count += do_remove(UNIHASH_TABLE_COLUMNS, "unihashes_v2", cursor) self.db.commit() - self.write_message({}) + self.write_message({"count": count}) def query_equivalent(self, cursor, method, taskhash): # This is part of the inner loop and must be as fast as possible From patchwork Thu Sep 28 17:05:50 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joshua Watt X-Patchwork-Id: 31324 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 65454E732FD for ; Thu, 28 Sep 2023 17:06:09 +0000 (UTC) Received: from mail-oi1-f179.google.com (mail-oi1-f179.google.com [209.85.167.179]) by mx.groups.io with SMTP id smtpd.web10.19056.1695920763629337148 for ; Thu, 28 Sep 2023 10:06:03 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=UwtIZkpT; spf=pass (domain: gmail.com, ip: 209.85.167.179, mailfrom: jpewhacker@gmail.com) Received: by mail-oi1-f179.google.com with SMTP id 5614622812f47-3ae2ec1a222so6193352b6e.2 for ; Thu, 28 Sep 2023 10:06:03 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1695920762; x=1696525562; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=UySHwEmjdXPUeKVRH4m+lSRLlgKpZJaTD/DUPenGqgM=; b=UwtIZkpT8CmBduLHkgOhBZzDhRo4AOvin9sk/I+Tm+y1d0ixDuDE7E2Ua4ZOry5jTu 18XpLpQKndB8bz/6sHDPE9yhcrRQwEY9Uq3PvRdLxqHExskVDa/8X8mkXxFsxMEVx62y gZTCXqzdZ7URIQJB/oADd1dDpXJQ8imVTpDKJg2bG2JOuOZbszNqBR2Zy0KYVNc1xxx3 GaYzrZhkNe5Nea9oBqy+rpu0z7nplLb6w8rLV0ZnYmG42EDgP7qxi/dp2QfcMqww+WgZ H+oIROkJYC8djRN6Q37l9RnYKJgsDIraauEfuyG57lxhTtgi55dD43FuH6PG/e1CYb7+ SMhg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1695920762; x=1696525562; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=UySHwEmjdXPUeKVRH4m+lSRLlgKpZJaTD/DUPenGqgM=; b=HyQoLJagyg6nl0a0XYR3yexrtcrzTJwJpx9u1ZllQlWEPo4/HRtDS9AgO/Xi4jDEIg k8za/CV0brLy2I9BweYqAIrHLRpmXScbOw0lHwIxC0inn0oJoEkenJC80JfpkkKZWuO8 RYB+zEaej3pDUp7aOGkaGb13WskW4NwKCT4wQtV6v3xamnZP7A2Lfx9vMEBphL9CASfD RcWCLR+k9Say25uGg5t9AJLGpaXOJFC0iC9TpxZqDgJiX4b/KYYKFOdxr7PMC2bLdgPK bd6xPN1VwpXAxqggFiCgLaEYrnnpRxbJLTC2hysKId5xWznWav/lR/gKqfhy+glR7X16 DCcA== X-Gm-Message-State: AOJu0Yw7jeEPrpwher+aUe/3htDFfugC89F67/eILAW6u6kcWUKIuXSh 6GuQFpcbdCMK8pBC0l6bO9dxKhR0pXg= X-Google-Smtp-Source: AGHT+IHxhNadFIZT5RA0Aqkhtg6Tjfar9yqd8VXeQJZABBMyvfoRKzGdp9/gc6L8wcU/E5aRn5jXgg== X-Received: by 2002:a05:6870:d38e:b0:1d0:c524:c05b with SMTP id k14-20020a056870d38e00b001d0c524c05bmr1973086oag.38.1695920762135; Thu, 28 Sep 2023 10:06:02 -0700 (PDT) Received: from localhost.localdomain ([2601:282:4300:19e0::f27c]) by smtp.gmail.com with ESMTPSA id h3-20020a056870860300b001dd533c4bd1sm1389772oal.1.2023.09.28.10.06.00 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 28 Sep 2023 10:06:01 -0700 (PDT) From: Joshua Watt X-Google-Original-From: Joshua Watt To: bitbake-devel@lists.openembedded.org Cc: Joshua Watt Subject: [bitbake-devel][RFC 4/5] hashserv: tests: Add external database tests Date: Thu, 28 Sep 2023 11:05:50 -0600 Message-Id: <20230928170551.4193224-5-JPEWhacker@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230928170551.4193224-1-JPEWhacker@gmail.com> References: <20230928170551.4193224-1-JPEWhacker@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 28 Sep 2023 17:06:09 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/15146 Adds support for running the hash equivalence test suite against an external hash equivalence implementation. Signed-off-by: Joshua Watt --- bitbake/lib/hashserv/tests.py | 53 ++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index 5f1ad585072..395fa451879 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py @@ -51,13 +51,20 @@ class HashEquivalenceTestSetup(object): server.serve_as_process(prefunc=prefunc, args=(self.server_index,)) self.addCleanup(cleanup_server, server) + return server + + def start_client(self, server_address): def cleanup_client(client): client.close() - client = create_client(server.address) + client = create_client(server_address) self.addCleanup(cleanup_client, client) - return (client, server) + return client + + def start_test_server(self): + server = self.start_server() + return server.address def setUp(self): if sys.version_info < (3, 5, 0): @@ -66,7 +73,9 @@ class HashEquivalenceTestSetup(object): self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv') self.addCleanup(self.temp_dir.cleanup) - (self.client, self.server) = self.start_server() + self.server_address = self.start_test_server() + + self.client = self.start_client(self.server_address) def assertClientGetHash(self, client, taskhash, unihash): result = client.get_unihash(self.METHOD, taskhash) @@ -183,7 +192,7 @@ class HashEquivalenceCommonTests(object): def test_stress(self): def query_server(failures): - client = Client(self.server.address) + client = Client(self.server_address) try: for i in range(1000): taskhash = hashlib.sha256() @@ -222,8 +231,10 @@ class HashEquivalenceCommonTests(object): # the side client. It also verifies that the results are pulled into # the downstream database by checking that the downstream and side servers # match after the downstream is done waiting for all backfill tasks - (down_client, down_server) = self.start_server(upstream=self.server.address) - (side_client, side_server) = self.start_server(dbpath=down_server.dbpath) + down_server = self.start_server(upstream=self.server_address) + down_client = self.start_client(down_server.address) + side_server = self.start_server(dbpath=down_server.dbpath) + side_client = self.start_client(side_server.address) def check_hash(taskhash, unihash, old_sidehash): nonlocal down_client @@ -328,14 +339,18 @@ class HashEquivalenceCommonTests(object): self.assertEqual(result['method'], self.METHOD) def test_ro_server(self): - (ro_client, ro_server) = self.start_server(dbpath=self.server.dbpath, read_only=True) + rw_server = self.start_server() + rw_client = self.start_client(rw_server.address) + + ro_server = self.start_server(dbpath=rw_server.dbpath, read_only=True) + ro_client = self.start_client(ro_server.address) # Report a hash via the read-write server taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' - result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) + result = rw_client.report_unihash(taskhash, self.METHOD, outhash, unihash) self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') # Check the hash via the read-only server @@ -350,7 +365,7 @@ class HashEquivalenceCommonTests(object): ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2) # Ensure that the database was not modified - self.assertClientGetHash(self.client, taskhash2, None) + self.assertClientGetHash(rw_client, taskhash2, None) def test_slow_server_start(self): @@ -370,7 +385,7 @@ class HashEquivalenceCommonTests(object): old_signal = signal.signal(signal.SIGTERM, do_nothing) self.addCleanup(signal.signal, signal.SIGTERM, old_signal) - _, server = self.start_server(prefunc=prefunc) + server = self.start_server(prefunc=prefunc) server.process.terminate() time.sleep(30) event.set() @@ -460,3 +475,21 @@ class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceComm # If IPv6 is enabled, it should be safe to use localhost directly, in general # case it is more reliable to resolve the IP address explicitly. return socket.gethostbyname("localhost") + ":0" + +class TestHashEquivalenceExternalServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): + def start_test_server(self): + if 'BB_TEST_HASHSERV' not in os.environ: + self.skipTest('BB_TEST_HASHSERV not defined to test an external server') + + return os.environ['BB_TEST_HASHSERV'] + + def start_server(self, *args, **kwargs): + self.skipTest('Cannot start local server when testing external servers') + + def setUp(self): + super().setUp() + self.client.remove({"method": self.METHOD}) + + def tearDown(self): + self.client.remove({"method": self.METHOD}) + super().tearDown() From patchwork Thu Sep 28 17:05:51 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Joshua Watt X-Patchwork-Id: 31323 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from aws-us-west-2-korg-lkml-1.web.codeaurora.org (localhost.localdomain [127.0.0.1]) by smtp.lore.kernel.org (Postfix) with ESMTP id 69486E732FE for ; Thu, 28 Sep 2023 17:06:09 +0000 (UTC) Received: from mail-oa1-f49.google.com (mail-oa1-f49.google.com [209.85.160.49]) by mx.groups.io with SMTP id smtpd.web10.19057.1695920765151788179 for ; Thu, 28 Sep 2023 10:06:05 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20230601 header.b=KVkg1vbh; spf=pass (domain: gmail.com, ip: 209.85.160.49, mailfrom: jpewhacker@gmail.com) Received: by mail-oa1-f49.google.com with SMTP id 586e51a60fabf-1dcfb2a3282so5067149fac.2 for ; Thu, 28 Sep 2023 10:06:05 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1695920764; x=1696525564; darn=lists.openembedded.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=TgcnrNjvCa+O23jJRcO2GfBxbQCQ2CJH/XeNs8oGsOs=; b=KVkg1vbhc3Y0QhjOZkyeH5FzWW0Fmm5hHQ4LXZFTe2GTrz6H53mFZ8fnfpE25ha/QY tjaoH91h8s3kTv4GfKWpcosRtLaVaq2ULdwd0LZUoiQGyatkzeMKMNLdn6UVJIRkJwYc 2KWBfZomuAPnBq2sUY1lCrtoVNkwf86f4X5rJz3+IWRLiJC0bA6AVhROx7n06OmrnCdx aJhl1aa0xTI/Hsof50gVcBp3pU2obGc71kc7GO8jPuiYOdMXJBCRTcOhcrtCv8i1iC+d f+JiW8m6iAJXBW1IW6r7geAjBpJsR1vrY8m9n08lOAG6R5vxYjAze3/IHs8+8/+thJEL +Wyw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1695920764; x=1696525564; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=TgcnrNjvCa+O23jJRcO2GfBxbQCQ2CJH/XeNs8oGsOs=; b=dUtj7TK6AfTe5WexHT700JjtY8IPJYHqHJULD7PNTniefX2GxP9LehoJuZrcmLdPGE u4sEVx7rl78GnHTpECj9ODJ4FM7gzZ2yLfRLoyNzQq/1QlqbB5RcUxULxD7frU+tomKH +Fku8Au6220YK7BaGnP5p057/XdL7TmQIERkfffpCLSKIElYLRhf96T+L2USVYxVYWZ9 vdssVKjHmyAoXjwdLrAahAVfXKX6V/hdnThu5aooq/4yla/YwhlX7MQfeSOrvS+saiDA BIqqPXZ7DmHkjUsdRMXw/0XIjmgT/GfjuHMeUdi6rOuz40ArqORBh4eDEs4c3Pdx1zlk QAAQ== X-Gm-Message-State: AOJu0Yz5amcFrtyOkDoyP69WuEqEZe8nhmw3aAu+CazlbyOMdtZlL+4I d/is0Xl+zNcFI77vwC4CIRjDK4JncnU= X-Google-Smtp-Source: AGHT+IGri4lGwvotJcShi8OoP/b7VvfSyN2J3WUweha0lJoIIEW1pa0PVNCHHIOcMEHEY4hMYrrSkA== X-Received: by 2002:a05:6870:958b:b0:1bf:df47:7b5e with SMTP id k11-20020a056870958b00b001bfdf477b5emr1907390oao.16.1695920763774; Thu, 28 Sep 2023 10:06:03 -0700 (PDT) Received: from localhost.localdomain ([2601:282:4300:19e0::f27c]) by smtp.gmail.com with ESMTPSA id h3-20020a056870860300b001dd533c4bd1sm1389772oal.1.2023.09.28.10.06.02 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 28 Sep 2023 10:06:02 -0700 (PDT) From: Joshua Watt X-Google-Original-From: Joshua Watt To: bitbake-devel@lists.openembedded.org Cc: Joshua Watt Subject: [bitbake-devel][RFC 5/5] hashserv: Add websocket connection implementation Date: Thu, 28 Sep 2023 11:05:51 -0600 Message-Id: <20230928170551.4193224-6-JPEWhacker@gmail.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230928170551.4193224-1-JPEWhacker@gmail.com> References: <20230928170551.4193224-1-JPEWhacker@gmail.com> MIME-Version: 1.0 List-Id: X-Webhook-Received: from li982-79.members.linode.com [45.33.32.79] by aws-us-west-2-korg-lkml-1.web.codeaurora.org with HTTPS for ; Thu, 28 Sep 2023 17:06:09 -0000 X-Groupsio-URL: https://lists.openembedded.org/g/bitbake-devel/message/15147 Adds support for the hash equivalence client to talk to a server over a websocket. Since websocket are already message orientated (as opposed to stream orientated like the other implementations), a new MessageParser() class is implemented for them (although, this class would work for any message orientation socket, not just websockets). Note that websocket support does require the 3rd party websockets python module be installed on the host, but it should not be required unless websockets are actually being used. Signed-off-by: Joshua Watt --- bitbake/lib/bb/asyncrpc/client.py | 38 +++++++++++++++++++++++++++++++ bitbake/lib/hashserv/__init__.py | 9 ++++++++ bitbake/lib/hashserv/client.py | 1 + 3 files changed, 48 insertions(+) diff --git a/bitbake/lib/bb/asyncrpc/client.py b/bitbake/lib/bb/asyncrpc/client.py index 335da09d8c6..b0892a82436 100644 --- a/bitbake/lib/bb/asyncrpc/client.py +++ b/bitbake/lib/bb/asyncrpc/client.py @@ -78,6 +78,35 @@ class StreamParser(object): self.writer = None +class MessageParser(object): + def __init__(self, socket, timeout): + self.socket = socket + self.timeout = timeout + + async def setup_connection(self, proto_name, proto_version): + await self.socket.send("%s %s" % (proto_name, proto_version)) + + async def invoke(self, msg): + await self.socket.send(json.dumps(msg)) + try: + message = await asyncio.wait_for(self.socket.recv(), self.timeout) + except asyncio.TimeoutError: + raise ConnectionError("Timed out waiting for server") + + return json.loads(message) + + async def close(self): + if self.socket is not None: + await self.socket.close() + self.socket = None + + async def send(self, msg): + await self.socket.send(msg) + + async def recv(self): + return await self.socket.recv() + + class AsyncClient(object): def __init__(self, proto_name, proto_version, logger, timeout=30): self.socket = None @@ -111,6 +140,15 @@ class AsyncClient(object): self._connect_sock = connect_sock + async def connect_websocket(self, uri): + import websockets + + async def connect_sock(): + websocket = await websockets.connect(uri) + return MessageParser(websocket, self.timeout) + + self._connect_sock = connect_sock + async def setup_connection(self): await self.socket.setup_connection(self.proto_name, self.proto_version) diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py index 9cb3fd57a51..aa5c7e736d7 100644 --- a/bitbake/lib/hashserv/__init__.py +++ b/bitbake/lib/hashserv/__init__.py @@ -11,9 +11,12 @@ import itertools import json UNIX_PREFIX = "unix://" +WS_PREFIX = "ws://" +WSS_PREFIX = "wss://" ADDR_TYPE_UNIX = 0 ADDR_TYPE_TCP = 1 +ADDR_TYPE_WS = 2 # The Python async server defaults to a 64K receive buffer, so we hardcode our # maximum chunk size. It would be better if the client and server reported to @@ -91,6 +94,8 @@ def setup_database(database, sync=True): def parse_address(addr): if addr.startswith(UNIX_PREFIX): return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX):],)) + elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX): + return (ADDR_TYPE_WS, (addr,)) else: m = re.match(r'\[(?P[^\]]*)\]:(?P\d+)$', addr) if m is not None: @@ -137,6 +142,8 @@ def create_client(addr): (typ, a) = parse_address(addr) if typ == ADDR_TYPE_UNIX: c.connect_unix(*a) + elif typ == ADDR_TYPE_WS: + c.connect_websocket(*a) else: c.connect_tcp(*a) @@ -149,6 +156,8 @@ async def create_async_client(addr): (typ, a) = parse_address(addr) if typ == ADDR_TYPE_UNIX: await c.connect_unix(*a) + elif typ == ADDR_TYPE_WS: + await c.connect_websocket(*a) else: await c.connect_tcp(*a) diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py index 7d2b9cb394f..c80aa32fa55 100644 --- a/bitbake/lib/hashserv/client.py +++ b/bitbake/lib/hashserv/client.py @@ -107,6 +107,7 @@ class Client(bb.asyncrpc.Client): super().__init__() self._add_methods( "connect_tcp", + "connect_websocket", "get_unihash", "report_unihash", "report_unihash_equiv",