History

scripts/jobserver-exec: move its class to the lib directory

To make it easier to be re-used, move the JobserverExec class to the library directory.

Mauro Carvalho Chehab committed 1 year ago

Commit fce6df71 parent 2a14f022 files · +160 −141Browse files

Showing the changes to scripts (2 of 2 files). Show all

Unified · Split

2 files
  1. scripts/jobserver-exec +11 −141
  2. scripts/lib/jobserver.py +149 −0
scripts/jobserver-exec+11−141
@@ -1,155 +1,25 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0+2# SPDX-License-Identifier: GPL-2.0+
3#
4# pylint: disable=C0103,C0209
5#
6# This determines how many parallel tasks "make" is expecting, as it is
7# not exposed via an special variables, reserves them all, runs a subprocess
8# with PARALLELISM environment variable set, and releases the jobs back again.
9#
10# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
11 3
12"""
13Interacts with the POSIX jobserver during the Kernel build time.
14
15A "normal" jobserver task, like the one initiated by a make subrocess would do:
16
17 - open read/write file descriptors to communicate with the job server;
18 - ask for one slot by calling:
19 claim = os.read(reader, 1)
20 - when the job finshes, call:
21 os.write(writer, b"+") # os.write(writer, claim)
22
23Here, the goal is different: This script aims to get the remaining number
24of slots available, using all of them to run a command which handle tasks in
25parallel. To to that, it has a loop that ends only after there are no
26slots left. It then increments the number by one, in order to allow a
27call equivalent to make -j$((claim+1)), e.g. having a parent make creating
28$claim child to do the actual work.
29
30The end goal here is to keep the total number of build tasks under the
31limit established by the initial make -j$n_proc call.
32"""
33
34import errno
35import os4import os
36import subprocess
37import sys5import sys
38 6
7LIB_DIR = "lib"
8SRC_DIR = os.path.dirname(os.path.realpath(__file__))
39 9
40class JobserverExec:10sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR))
41 """
42 Claim all slots from make using POSIX Jobserver.
43
44 The main methods here are:
45 - open(): reserves all slots;
46 - close(): method returns all used slots back to make;
47 - run(): executes a command setting PARALLELISM=<available slots jobs + 1>
48 """
49
50 def __init__(self):
51 """Initialize internal vars"""
52 self.claim = 0
53 self.jobs = b""
54 self.reader = None
55 self.writer = None
56 self.is_open = False
57
58 def open(self):
59 """Reserve all available slots to be claimed later on"""
60
61 if self.is_open:
62 return
63
64 try:
65 # Fetch the make environment options.
66 flags = os.environ["MAKEFLAGS"]
67 # Look for "--jobserver=R,W"
68 # Note that GNU Make has used --jobserver-fds and --jobserver-auth
69 # so this handles all of them.
70 opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
71
72 # Parse out R,W file descriptor numbers and set them nonblocking.
73 # If the MAKEFLAGS variable contains multiple instances of the
74 # --jobserver-auth= option, the last one is relevant.
75 fds = opts[-1].split("=", 1)[1]
76
77 # Starting with GNU Make 4.4, named pipes are used for reader
78 # and writer.
79 # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134
80 _, _, path = fds.partition("fifo:")
81 11
82 if path:12from jobserver import JobserverExec # pylint: disable=C0415
83 self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
84 self.writer = os.open(path, os.O_WRONLY)
85 else:
86 self.reader, self.writer = [int(x) for x in fds.split(",", 1)]
87 # Open a private copy of reader to avoid setting nonblocking
88 # on an unexpecting process with the same reader fd.
89 self.reader = os.open("/proc/self/fd/%d" % (self.reader),
90 os.O_RDONLY | os.O_NONBLOCK)
91 13
92 # Read out as many jobserver slots as possible
93 while True:
94 try:
95 slot = os.read(self.reader, 8)
96 self.jobs += slot
97 except (OSError, IOError) as e:
98 if e.errno == errno.EWOULDBLOCK:
99 # Stop at the end of the jobserver queue.
100 break
101 # If something went wrong, give back the jobs.
102 if self.jobs:
103 os.write(self.writer, self.jobs)
104 raise e
105 14
106 # Add a bump for our caller's reserveration, since we're just going15"""
107 # to sit here blocked on our child.16Determines how many parallel tasks "make" is expecting, as it is
108 self.claim = len(self.jobs) + 117not exposed via an special variables, reserves them all, runs a subprocess
109 18with PARALLELISM environment variable set, and releases the jobs back again.
110 except (KeyError, IndexError, ValueError, OSError, IOError):
111 # Any missing environment strings or bad fds should result in just
112 # not being parallel.
113 self.claim = None
114
115 self.is_open = True
116
117 def close(self):
118 """Return all reserved slots to Jobserver"""
119
120 if not self.is_open:
121 return
122
123 # Return all the reserved slots.
124 if len(self.jobs):
125 os.write(self.writer, self.jobs)
126
127 self.is_open = False
128
129 def __enter__(self):
130 self.open()
131 return self
132
133 def __exit__(self, exc_type, exc_value, exc_traceback):
134 self.close()
135
136 def run(self, cmd):
137 """
138 Run a command setting PARALLELISM env variable to the number of
139 available job slots (claim) + 1, e.g. it will reserve claim slots
140 to do the actual build work, plus one to monitor its children.
141 """
142 self.open() # Ensure that self.claim is set
143
144 # We can only claim parallelism if there was a jobserver (i.e. a
145 # top-level "-jN" argument) and there were no other failures. Otherwise
146 # leave out the environment variable and let the child figure out what
147 # is best.
148 if self.claim:
149 os.environ["PARALLELISM"] = str(self.claim)
150
151 return subprocess.call(cmd)
152 19
20See:
21 https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
22"""
153 23
154def main():24def main():
155 """Main program"""25 """Main program"""
scripts/lib/jobserver.py+149−0
@@ -0,0 +1,149 @@
1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0+
3#
4# pylint: disable=C0103,C0209
5#
6#
7
8"""
9Interacts with the POSIX jobserver during the Kernel build time.
10
11A "normal" jobserver task, like the one initiated by a make subrocess would do:
12
13 - open read/write file descriptors to communicate with the job server;
14 - ask for one slot by calling:
15 claim = os.read(reader, 1)
16 - when the job finshes, call:
17 os.write(writer, b"+") # os.write(writer, claim)
18
19Here, the goal is different: This script aims to get the remaining number
20of slots available, using all of them to run a command which handle tasks in
21parallel. To to that, it has a loop that ends only after there are no
22slots left. It then increments the number by one, in order to allow a
23call equivalent to make -j$((claim+1)), e.g. having a parent make creating
24$claim child to do the actual work.
25
26The end goal here is to keep the total number of build tasks under the
27limit established by the initial make -j$n_proc call.
28
29See:
30 https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
31"""
32
33import errno
34import os
35import subprocess
36import sys
37
38class JobserverExec:
39 """
40 Claim all slots from make using POSIX Jobserver.
41
42 The main methods here are:
43 - open(): reserves all slots;
44 - close(): method returns all used slots back to make;
45 - run(): executes a command setting PARALLELISM=<available slots jobs + 1>
46 """
47
48 def __init__(self):
49 """Initialize internal vars"""
50 self.claim = 0
51 self.jobs = b""
52 self.reader = None
53 self.writer = None
54 self.is_open = False
55
56 def open(self):
57 """Reserve all available slots to be claimed later on"""
58
59 if self.is_open:
60 return
61
62 try:
63 # Fetch the make environment options.
64 flags = os.environ["MAKEFLAGS"]
65 # Look for "--jobserver=R,W"
66 # Note that GNU Make has used --jobserver-fds and --jobserver-auth
67 # so this handles all of them.
68 opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
69
70 # Parse out R,W file descriptor numbers and set them nonblocking.
71 # If the MAKEFLAGS variable contains multiple instances of the
72 # --jobserver-auth= option, the last one is relevant.
73 fds = opts[-1].split("=", 1)[1]
74
75 # Starting with GNU Make 4.4, named pipes are used for reader
76 # and writer.
77 # Example argument: --jobserver-auth=fifo:/tmp/GMfifo8134
78 _, _, path = fds.partition("fifo:")
79
80 if path:
81 self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
82 self.writer = os.open(path, os.O_WRONLY)
83 else:
84 self.reader, self.writer = [int(x) for x in fds.split(",", 1)]
85 # Open a private copy of reader to avoid setting nonblocking
86 # on an unexpecting process with the same reader fd.
87 self.reader = os.open("/proc/self/fd/%d" % (self.reader),
88 os.O_RDONLY | os.O_NONBLOCK)
89
90 # Read out as many jobserver slots as possible
91 while True:
92 try:
93 slot = os.read(self.reader, 8)
94 self.jobs += slot
95 except (OSError, IOError) as e:
96 if e.errno == errno.EWOULDBLOCK:
97 # Stop at the end of the jobserver queue.
98 break
99 # If something went wrong, give back the jobs.
100 if self.jobs:
101 os.write(self.writer, self.jobs)
102 raise e
103
104 # Add a bump for our caller's reserveration, since we're just going
105 # to sit here blocked on our child.
106 self.claim = len(self.jobs) + 1
107
108 except (KeyError, IndexError, ValueError, OSError, IOError):
109 # Any missing environment strings or bad fds should result in just
110 # not being parallel.
111 self.claim = None
112
113 self.is_open = True
114
115 def close(self):
116 """Return all reserved slots to Jobserver"""
117
118 if not self.is_open:
119 return
120
121 # Return all the reserved slots.
122 if len(self.jobs):
123 os.write(self.writer, self.jobs)
124
125 self.is_open = False
126
127 def __enter__(self):
128 self.open()
129 return self
130
131 def __exit__(self, exc_type, exc_value, exc_traceback):
132 self.close()
133
134 def run(self, cmd, *args, **pwargs):
135 """
136 Run a command setting PARALLELISM env variable to the number of
137 available job slots (claim) + 1, e.g. it will reserve claim slots
138 to do the actual build work, plus one to monitor its children.
139 """
140 self.open() # Ensure that self.claim is set
141
142 # We can only claim parallelism if there was a jobserver (i.e. a
143 # top-level "-jN" argument) and there were no other failures. Otherwise
144 # leave out the environment variable and let the child figure out what
145 # is best.
146 if self.claim:
147 os.environ["PARALLELISM"] = str(self.claim)
148
149 return subprocess.call(cmd, *args, **pwargs)