scripts/jobserver-exec: move its class to the lib directory

To make it easier to be re-used, move the JobserverExec class to the library directory.

Mauro Carvalho Chehab committed 1 year ago

Commit fce6df71 parent 2a14f022 files · +160 −141Browse files

Unified · Split

2 files

scripts/jobserver-exec +11 −141
scripts/lib/jobserver.py +149 −0

@@ -1,155 +1,25 @@
1	1	#!/usr/bin/env python3
2	2	# SPDX-License-Identifier: GPL-2.0+
3		−#
4		−# pylint: disable=C0103,C0209
5		−#
6		−# This determines how many parallel tasks "make" is expecting, as it is
7		−# not exposed via an special variables, reserves them all, runs a subprocess
8		−# with PARALLELISM environment variable set, and releases the jobs back again.
9		−#
10		−# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
11	3
12		−"""
13		−Interacts with the POSIX jobserver during the Kernel build time.
14		−
15		−A "normal" jobserver task, like the one initiated by a make subrocess would do:
16		−
17		− - open read/write file descriptors to communicate with the job server;
18		− - ask for one slot by calling:
19		− claim = os.read(reader, 1)
20		− - when the job finshes, call:
21		− os.write(writer, b"+") # os.write(writer, claim)
22		−
23		−Here, the goal is different: This script aims to get the remaining number
24		−of slots available, using all of them to run a command which handle tasks in
25		−parallel. To to that, it has a loop that ends only after there are no
26		−slots left. It then increments the number by one, in order to allow a
27		−call equivalent to make -j$((claim+1)), e.g. having a parent make creating
28		−$claim child to do the actual work.
29		−
30		−The end goal here is to keep the total number of build tasks under the
31		−limit established by the initial make -j$n_proc call.
32		−"""
33		−
34		−import errno
35	4	import os
36		−import subprocess
37	5	import sys
38	6
	7	+LIB_DIR = "lib"
	8	+SRC_DIR = os.path.dirname(os.path.realpath(__file__))
39	9
40		−class JobserverExec:
41		− """
42		− Claim all slots from make using POSIX Jobserver.
43		−
44		− The main methods here are:
45		− - open(): reserves all slots;
46		− - close(): method returns all used slots back to make;
47		− - run(): executes a command setting PARALLELISM=<available slots jobs + 1>
48		− """
49		−
50		− def __init__(self):
51		− """Initialize internal vars"""
52		− self.claim = 0
53		− self.jobs = b""
54		− self.reader = None
55		− self.writer = None
56		− self.is_open = False
57		−
58		− def open(self):
59		− """Reserve all available slots to be claimed later on"""
60		−
61		− if self.is_open:
62		− return
63		−
64		− try:
65		− # Fetch the make environment options.
66		− flags = os.environ["MAKEFLAGS"]
67		− # Look for "--jobserver=R,W"
68		− # Note that GNU Make has used --jobserver-fds and --jobserver-auth
69		− # so this handles all of them.
70		− opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
71		−
72		− # Parse out R,W file descriptor numbers and set them nonblocking.
73		− # If the MAKEFLAGS variable contains multiple instances of the
74		− # --jobserver-auth= option, the last one is relevant.
75		− fds = opts[-1].split("=", 1)[1]
76		−
77		− # Starting with GNU Make 4.4, named pipes are used for reader
78		− # and writer.
79		− # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134
80		− _, _, path = fds.partition("fifo:")
	10	+sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
81	11
82		− if path:
83		− self.reader = os.open(path, os.O_RDONLY \| os.O_NONBLOCK)
84		− self.writer = os.open(path, os.O_WRONLY)
85		− else:
86		− self.reader, self.writer = [int(x) for x in fds.split(",", 1)]
87		− # Open a private copy of reader to avoid setting nonblocking
88		− # on an unexpecting process with the same reader fd.
89		− self.reader = os.open("/proc/self/fd/%d" % (self.reader),
90		− os.O_RDONLY \| os.O_NONBLOCK)
	12	+from jobserver import JobserverExec # pylint: disable=C0415
91	13
92		− # Read out as many jobserver slots as possible
93		− while True:
94		− try:
95		− slot = os.read(self.reader, 8)
96		− self.jobs += slot
97		− except (OSError, IOError) as e:
98		− if e.errno == errno.EWOULDBLOCK:
99		− # Stop at the end of the jobserver queue.
100		− break
101		− # If something went wrong, give back the jobs.
102		− if self.jobs:
103		− os.write(self.writer, self.jobs)
104		− raise e
105	14
106		− # Add a bump for our caller's reserveration, since we're just going
107		− # to sit here blocked on our child.
108		− self.claim = len(self.jobs) + 1
109		−
110		− except (KeyError, IndexError, ValueError, OSError, IOError):
111		− # Any missing environment strings or bad fds should result in just
112		− # not being parallel.
113		− self.claim = None
114		−
115		− self.is_open = True
116		−
117		− def close(self):
118		− """Return all reserved slots to Jobserver"""
119		−
120		− if not self.is_open:
121		− return
122		−
123		− # Return all the reserved slots.
124		− if len(self.jobs):
125		− os.write(self.writer, self.jobs)
126		−
127		− self.is_open = False
128		−
129		− def __enter__(self):
130		− self.open()
131		− return self
132		−
133		− def __exit__(self, exc_type, exc_value, exc_traceback):
134		− self.close()
135		−
136		− def run(self, cmd):
137		− """
138		− Run a command setting PARALLELISM env variable to the number of
139		− available job slots (claim) + 1, e.g. it will reserve claim slots
140		− to do the actual build work, plus one to monitor its children.
141		− """
142		− self.open() # Ensure that self.claim is set
143		−
144		− # We can only claim parallelism if there was a jobserver (i.e. a
145		− # top-level "-jN" argument) and there were no other failures. Otherwise
146		− # leave out the environment variable and let the child figure out what
147		− # is best.
148		− if self.claim:
149		− os.environ["PARALLELISM"] = str(self.claim)
150		−
151		− return subprocess.call(cmd)
	15	+"""
	16	+Determines how many parallel tasks "make" is expecting, as it is
	17	+not exposed via an special variables, reserves them all, runs a subprocess
	18	+with PARALLELISM environment variable set, and releases the jobs back again.
152	19
	20	+See:
	21	+ https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
	22	+"""
153	23
154	24	def main():
155	25	"""Main program"""

scripts/lib/jobserver.py+149−0

@@ -0,0 +1,149 @@
	1	+#!/usr/bin/env python3
	2	+# SPDX-License-Identifier: GPL-2.0+
	3	+#
	4	+# pylint: disable=C0103,C0209
	5	+#
	6	+#
	7	+
	8	+"""
	9	+Interacts with the POSIX jobserver during the Kernel build time.
	10	+
	11	+A "normal" jobserver task, like the one initiated by a make subrocess would do:
	12	+
	13	+ - open read/write file descriptors to communicate with the job server;
	14	+ - ask for one slot by calling:
	15	+ claim = os.read(reader, 1)
	16	+ - when the job finshes, call:
	17	+ os.write(writer, b"+") # os.write(writer, claim)
	18	+
	19	+Here, the goal is different: This script aims to get the remaining number
	20	+of slots available, using all of them to run a command which handle tasks in
	21	+parallel. To to that, it has a loop that ends only after there are no
	22	+slots left. It then increments the number by one, in order to allow a
	23	+call equivalent to make -j$((claim+1)), e.g. having a parent make creating
	24	+$claim child to do the actual work.
	25	+
	26	+The end goal here is to keep the total number of build tasks under the
	27	+limit established by the initial make -j$n_proc call.
	28	+
	29	+See:
	30	+ https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
	31	+"""
	32	+
	33	+import errno
	34	+import os
	35	+import subprocess
	36	+import sys
	37	+
	38	+class JobserverExec:
	39	+ """
	40	+ Claim all slots from make using POSIX Jobserver.
	41	+
	42	+ The main methods here are:
	43	+ - open(): reserves all slots;
	44	+ - close(): method returns all used slots back to make;
	45	+ - run(): executes a command setting PARALLELISM=<available slots jobs + 1>
	46	+ """
	47	+
	48	+ def __init__(self):
	49	+ """Initialize internal vars"""
	50	+ self.claim = 0
	51	+ self.jobs = b""
	52	+ self.reader = None
	53	+ self.writer = None
	54	+ self.is_open = False
	55	+
	56	+ def open(self):
	57	+ """Reserve all available slots to be claimed later on"""
	58	+
	59	+ if self.is_open:
	60	+ return
	61	+
	62	+ try:
	63	+ # Fetch the make environment options.
	64	+ flags = os.environ["MAKEFLAGS"]
	65	+ # Look for "--jobserver=R,W"
	66	+ # Note that GNU Make has used --jobserver-fds and --jobserver-auth
	67	+ # so this handles all of them.
	68	+ opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
	69	+
	70	+ # Parse out R,W file descriptor numbers and set them nonblocking.
	71	+ # If the MAKEFLAGS variable contains multiple instances of the
	72	+ # --jobserver-auth= option, the last one is relevant.
	73	+ fds = opts[-1].split("=", 1)[1]
	74	+
	75	+ # Starting with GNU Make 4.4, named pipes are used for reader
	76	+ # and writer.
	77	+ # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134
	78	+ _, _, path = fds.partition("fifo:")
	79	+
	80	+ if path:
	81	+ self.reader = os.open(path, os.O_RDONLY \| os.O_NONBLOCK)
	82	+ self.writer = os.open(path, os.O_WRONLY)
	83	+ else:
	84	+ self.reader, self.writer = [int(x) for x in fds.split(",", 1)]
	85	+ # Open a private copy of reader to avoid setting nonblocking
	86	+ # on an unexpecting process with the same reader fd.
	87	+ self.reader = os.open("/proc/self/fd/%d" % (self.reader),
	88	+ os.O_RDONLY \| os.O_NONBLOCK)
	89	+
	90	+ # Read out as many jobserver slots as possible
	91	+ while True:
	92	+ try:
	93	+ slot = os.read(self.reader, 8)
	94	+ self.jobs += slot
	95	+ except (OSError, IOError) as e:
	96	+ if e.errno == errno.EWOULDBLOCK:
	97	+ # Stop at the end of the jobserver queue.
	98	+ break
	99	+ # If something went wrong, give back the jobs.
	100	+ if self.jobs:
	101	+ os.write(self.writer, self.jobs)
	102	+ raise e
	103	+
	104	+ # Add a bump for our caller's reserveration, since we're just going
	105	+ # to sit here blocked on our child.
	106	+ self.claim = len(self.jobs) + 1
	107	+
	108	+ except (KeyError, IndexError, ValueError, OSError, IOError):
	109	+ # Any missing environment strings or bad fds should result in just
	110	+ # not being parallel.
	111	+ self.claim = None
	112	+
	113	+ self.is_open = True
	114	+
	115	+ def close(self):
	116	+ """Return all reserved slots to Jobserver"""
	117	+
	118	+ if not self.is_open:
	119	+ return
	120	+
	121	+ # Return all the reserved slots.
	122	+ if len(self.jobs):
	123	+ os.write(self.writer, self.jobs)
	124	+
	125	+ self.is_open = False
	126	+
	127	+ def __enter__(self):
	128	+ self.open()
	129	+ return self
	130	+
	131	+ def __exit__(self, exc_type, exc_value, exc_traceback):
	132	+ self.close()
	133	+
	134	+ def run(self, cmd, args, *pwargs):
	135	+ """
	136	+ Run a command setting PARALLELISM env variable to the number of
	137	+ available job slots (claim) + 1, e.g. it will reserve claim slots
	138	+ to do the actual build work, plus one to monitor its children.
	139	+ """
	140	+ self.open() # Ensure that self.claim is set
	141	+
	142	+ # We can only claim parallelism if there was a jobserver (i.e. a
	143	+ # top-level "-jN" argument) and there were no other failures. Otherwise
	144	+ # leave out the environment variable and let the child figure out what
	145	+ # is best.
	146	+ if self.claim:
	147	+ os.environ["PARALLELISM"] = str(self.claim)
	148	+
	149	+ return subprocess.call(cmd, args, *pwargs)