gen_compile_commands: move directory walk to a generator function

Currently, this script walks under the specified directory (default to the current directory), then parses all .cmd files found. Split it into a separate helper function because the next commit will add more helpers to pick up .cmd files associated with given file(s). There is no point to build and return a huge list at once. I used a generator so it works in the for-loop with less memory.

Masahiro Yamada committed 6 years ago

Commit fc2cb221 parent 6fca36f1 file · +32 −12Browse files

Unified · Split

scripts/gen_compile_commands.py+32−12

         log_level: A logging level to filter log output.
         directory: The work directory where the objects were built.
         output: Where to write the compile-commands JSON file.
+        paths: The list of directories to handle to find .cmd files.
     """
     usage = 'Creates a compile_commands.json database from kernel .cmd files'
     parser = argparse.ArgumentParser(description=usage)
     return (args.log_level,
             os.path.abspath(args.directory),
-            args.output)
+            args.output,
+            [args.directory])
+def cmdfiles_in_dir(directory):
+    """Generate the iterator of .cmd files found under the directory.
+    Walk under the given directory, and yield every .cmd file found.
+    Args:
+        directory: The directory to search for .cmd files.
+    Yields:
+        The path to a .cmd file.
+    """
+    filename_matcher = re.compile(_FILENAME_PATTERN)
+    for dirpath, _, filenames in os.walk(directory):
+        for filename in filenames:
+            if filename_matcher.match(filename):
+                yield os.path.join(dirpath, filename)
 def process_line(root_directory, command_prefix, file_path):
 def main():
     """Walks through the directory and finds and parses .cmd files."""
-    log_level, directory, output = parse_arguments()
+    log_level, directory, output, paths = parse_arguments()
     level = getattr(logging, log_level)
     logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
-    filename_matcher = re.compile(_FILENAME_PATTERN)
     line_matcher = re.compile(_LINE_PATTERN)
     compile_commands = []
-    for dirpath, _, filenames in os.walk(directory):
-        for filename in filenames:
-            if not filename_matcher.match(filename):
-                continue
-            filepath = os.path.join(dirpath, filename)
-            with open(filepath, 'rt') as f:
+    for path in paths:
+        cmdfiles = cmdfiles_in_dir(path)
+        for cmdfile in cmdfiles:
+            with open(cmdfile, 'rt') as f:
                 result = line_matcher.match(f.readline())
                 if result:
                     try:
-                        entry = process_line(directory,
+                        entry = process_line(directory, result.group(1),
-                                             result.group(1), result.group(2))
+                                             result.group(2))
                         compile_commands.append(entry)
                     except ValueError as err:
                         logging.info('Could not add line from %s: %s',
-                                     filepath, err)
+                                     cmdfile, err)
     with open(output, 'wt') as f:
         json.dump(compile_commands, f, indent=2, sort_keys=True)