diff --git a/contrib/bst-graph b/contrib/bst-graph deleted file mode 100755 index 3fe93e1ff..000000000 --- a/contrib/bst-graph +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Authors: -# Chandan Singh -# -'''Print dependency graph of given element(s) in DOT format. - -This script must be run from the same directory where you would normally -run `bst` commands. - -When `--format` option is specified, the output will also be rendered in the -given format. A file with name `bst-graph.{format}` will be created in the same -directory. To use this option, you must have the `graphviz` command line tool -installed. -''' - -import argparse -import subprocess -import re - -from graphviz import Digraph -from ruamel.yaml import YAML - -def parse_args(): - '''Handle parsing of command line arguments. - - Returns: - A argparse.Namespace object - ''' - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - 'element', nargs='*', - help='Name of the element' - ) - parser.add_argument( - '--format', - help='Redner the graph in given format (`pdf`, `png`, `svg` etc)' - ) - parser.add_argument( - '--view', action='store_true', - help='Open the rendered graph with the default application' - ) - return parser.parse_args() - - -def parse_graph(lines): - '''Return nodes and edges of the parsed grpah. - - Args: - lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS' - - Returns: - Tuple of format (nodes,build_deps,runtime_deps) - Each member of build_deps and runtime_deps is also a tuple. - ''' - parser = YAML(typ="safe") - nodes = set() - build_deps = set() - runtime_deps = set() - for line in lines: - line = line.strip() - if not line: - continue - # It is safe to split on '|' as it is not a valid character for - # element names. - name, build_dep, runtime_dep = line.split('|') - - build_dep = parser.load(build_dep) - runtime_dep = parser.load(runtime_dep) - - nodes.add(name) - [build_deps.add((name, dep)) for dep in build_dep if dep] - [runtime_deps.add((name, dep)) for dep in runtime_dep if dep] - - return nodes, build_deps, runtime_deps - - -def generate_graph(nodes, build_deps, runtime_deps): - '''Generate graph from given nodes and edges. - - Args: - nodes: set of nodes - build_deps: set of tuples of build depdencies - runtime_deps: set of tuples of runtime depdencies - - Returns: - A graphviz.Digraph object - ''' - graph = Digraph() - for node in nodes: - graph.node(node) - for source, target in build_deps: - graph.edge(source, target, label='build-dep') - for source, target in runtime_deps: - graph.edge(source, target, label='runtime-dep') - return graph - - -def main(): - args = parse_args() - cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}||'] - if 'element' in args: - cmd += args.element - graph_lines = subprocess.check_output(cmd, universal_newlines=True) - # NOTE: We generate nodes and edges before giving them to graphviz as - # the library does not de-deuplicate them. - nodes, build_deps, runtime_deps = parse_graph(re.split("\|\|", graph_lines)) - graph = generate_graph(nodes, build_deps, runtime_deps) - print(graph.source) - if args.format: - graph.render(cleanup=True, - filename='bst-graph', - format=args.format, - view=args.view) - - -if __name__ == '__main__': - main() diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 89475b5a1..38c621e05 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1654,3 +1654,24 @@ def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" with app.initialized(): app.stream.artifact_delete(artifacts, selection=deps) + + +################################################################### +# Graph Command # +################################################################### +@cli.command(short_help="Output pipeline dependency graph.") +@click.option( + "--format", + "-f", + "format_", + metavar="FORMAT", + default="dot", + type=click.STRING, + help="Output format: e.g. `dot` and `pkl`", +) +@click.argument("target", nargs=1, required=True, type=click.Path()) +@click.pass_obj +def graph(app, target, format_): + """Output dependency graph.""" + with app.initialized(): + app.stream.graph(target, format_) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 2266b27f0..d2774d02e 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -24,6 +24,7 @@ import shutil import tarfile import tempfile + from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable @@ -49,7 +50,7 @@ from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount from .plugin import Plugin -from . import utils, node, _yaml, _site, _pipeline +from . import utils, node, _tree, _yaml, _site, _pipeline # Stream() @@ -1228,6 +1229,36 @@ def redirect_element_names(self, elements): return list(output_elements) + # graph() + # + # Outputs a dependency graph for the target element, in the given format. + # The dot file format can be rendered using graphviz. + # + # Args: + # target (str): The target element from which to build a dependency graph. + # format_ ('dot'/'pkl'): A .pkl dictionary or graphviz .dot file. + # + def graph(self, target, format_): + def _render(scope): + tree = _tree.Tree() + scope_name = {_Scope.BUILD: 'buildtime', _Scope.RUN: 'runtime'}[scope] + + for element in self.load_selection([target], selection=_PipelineSelection.ALL, need_state=False): + dependencies = {d._get_full_name() for d in element._dependencies(scope, recurse=False) if d} + + for dep in dependencies: + tree.link(element._get_full_name(), dep) + + path = os.path.basename(target) + path, _ = os.path.splitext(path) + path = f'{path}.{scope_name}' + tree.save(path, format_) + + self._context.messenger.info(f'Rendered dependency graph: {path}') + + _render(_Scope.BUILD) + _render(_Scope.RUN) + # get_state() # # Get the State object owned by Stream diff --git a/src/buildstream/_tree.py b/src/buildstream/_tree.py new file mode 100644 index 000000000..b87019fe7 --- /dev/null +++ b/src/buildstream/_tree.py @@ -0,0 +1,42 @@ +import os +import pickle as pkl + +from typing import List + +class Tree: + def __init__(self): + self.nodes = set() + self.links = dict() + + def link(self, predecessor: str, successor: str) -> None: + self.links.setdefault(successor, []) + self.links.setdefault(predecessor, []) + + if successor == predecessor: + raise ValueError(f'Predecessor and successor are identical: {successor}={predecessor}') + elif successor in self.links[predecessor]: + raise ValueError(f'Cycle detected - not valid tree: {successor}<=>{predecessor}') + + for n in [predecessor, successor]: + self.nodes.add(n) + + self.links[successor].append(predecessor) + + def save(self, path: str, format_: str = 'dot') -> None: + """Saves the tree to disk. + + dot: A graphviz .dot file. + pkl: A treelib compatible dictionary .pkl file. + """ + with open(f'{path}.{format_}', 'w+') as file: + if format_ == 'pkl': + pkl.dump(self.links, file) + elif format_ == 'dot': + lines = [f'digraph "{path}" {{'] + lines.extend(f' "{n}" [label="{n}"]' for n in self.nodes) + lines.extend(f' "{p}" -> "{s}"' for s, ps in self.links.items() for p in ps) + lines.append(f'}}') + + file.writelines([f'{l}\n' for l in lines]) + else: + raise ValueError(f'Unknown format: {format_}')