Parse pipeline

parse_pipeline

SpecToken: TypeAlias = Token[SpecTokenKind] module-attribute

PassArgElementType = str | int | bool | float module-attribute

PassArgListType = tuple[PassArgElementType, ...] module-attribute

SpecTokenKind

Bases: Enum

Source code in xdsl/utils/parse_pipeline.py

class SpecTokenKind(Enum):
    EOF = object()

    IDENT = object()
    L_BRACE = "{"
    R_BRACE = "}"
    EQUALS = "="
    NUMBER = object()
    SPACE = object()
    STRING_LIT = object()
    MLIR_PIPELINE = object()
    COMMA = ","

EOF = object() class-attribute instance-attribute

IDENT = object() class-attribute instance-attribute

L_BRACE = '{' class-attribute instance-attribute

R_BRACE = '}' class-attribute instance-attribute

EQUALS = '=' class-attribute instance-attribute

NUMBER = object() class-attribute instance-attribute

SPACE = object() class-attribute instance-attribute

STRING_LIT = object() class-attribute instance-attribute

MLIR_PIPELINE = object() class-attribute instance-attribute

COMMA = ',' class-attribute instance-attribute

PipelineLexer

This tokenizes a pass declaration string: pipeline ::= pipeline-element (, pipeline-element) pipeline-element ::= MLIR_PIPELINE | pass-name options? options ::= { options-element ( options-element) } options-element ::= key (= value (, value)* )?

key ::= IDENT pass-name ::= IDENT value ::= NUMBER | BOOL | IDENT | STRING_LITERAL

Source code in xdsl/utils/parse_pipeline.py

class PipelineLexer:
    """
    This tokenizes a pass declaration string:
    pipeline          ::= pipeline-element (`,` pipeline-element)*
    pipeline-element  ::= MLIR_PIPELINE
                        | pass-name options?
    options           ::= `{` options-element ( ` ` options-element)* `}`
    options-element   ::= key (`=` value (`,` value)* )?

    key       ::= IDENT
    pass-name ::= IDENT
    value     ::= NUMBER | BOOL | IDENT | STRING_LITERAL
    """

    _stream: Iterator[SpecToken]
    _peeked: SpecToken | None

    def __init__(self, input_str: str):
        self._stream = PipelineLexer._generator(input_str)
        self._peeked = None

    @staticmethod
    def _generator(input_str: str) -> Iterator[SpecToken]:
        input = Input(input_str, "pass-pipeline")
        pos = 0
        end = len(input_str)

        if len(input_str) == 0:
            yield SpecToken(SpecTokenKind.EOF, Span(pos, pos + 1, input))
            return

        while True:
            token: SpecToken | None = None
            for pattern, kind in _lexer_rules:
                if (match := pattern.match(input_str, pos)) is not None:
                    token = SpecToken(kind, Span(match.start(), match.end(), input))
                    pos = match.end()
                    break
            if token is None:
                raise PassPipelineParseError(
                    SpecToken(SpecTokenKind.IDENT, Span(pos, pos + 1, input)),
                    "Unknown token",
                )
            yield token
            if pos >= end:
                yield SpecToken(SpecTokenKind.EOF, Span(pos, pos + 1, input))
                return

    def lex(self) -> SpecToken:
        token = self.peek()
        self._peeked = None
        return token

    def peek(self) -> SpecToken:
        if self._peeked is None:
            self._peeked = next(self._stream)
        return self._peeked

__init__(input_str: str)

Source code in xdsl/utils/parse_pipeline.py

def __init__(self, input_str: str):
    self._stream = PipelineLexer._generator(input_str)
    self._peeked = None

lex() -> SpecToken

Source code in xdsl/utils/parse_pipeline.py

def lex(self) -> SpecToken:
    token = self.peek()
    self._peeked = None
    return token

peek() -> SpecToken

Source code in xdsl/utils/parse_pipeline.py

def peek(self) -> SpecToken:
    if self._peeked is None:
        self._peeked = next(self._stream)
    return self._peeked

PipelinePassSpec dataclass

A pass name and its arguments.

Source code in xdsl/utils/parse_pipeline.py

@dataclass(eq=True, frozen=True)
class PipelinePassSpec:
    """
    A pass name and its arguments.
    """

    name: str
    args: dict[str, PassArgListType]

    def normalize_arg_names(self) -> PipelinePassSpec:
        """
        This normalized all arg names by replacing `-` with `_`
        """
        new_args: dict[str, PassArgListType] = dict()
        for k, v in self.args.items():
            new_args[k.replace("-", "_")] = v
        return PipelinePassSpec(name=self.name, args=new_args)

    def __str__(self) -> str:
        """
        This function returns a string containing the PipelineSpec name, its arguments
        and respective values for use on the commandline.
        """
        query = f"{self.name}"
        arguments_pipeline = " ".join(
            _pass_arg_list_type_str(arg_name, arg_val)
            for arg_name, arg_val in self.args.items()
        )
        query += f"{{{arguments_pipeline}}}" if self.args else ""

        return query

name: str instance-attribute

args: dict[str, PassArgListType] instance-attribute

__init__(name: str, args: dict[str, PassArgListType]) -> None

normalize_arg_names() -> PipelinePassSpec

This normalized all arg names by replacing - with _

Source code in xdsl/utils/parse_pipeline.py

def normalize_arg_names(self) -> PipelinePassSpec:
    """
    This normalized all arg names by replacing `-` with `_`
    """
    new_args: dict[str, PassArgListType] = dict()
    for k, v in self.args.items():
        new_args[k.replace("-", "_")] = v
    return PipelinePassSpec(name=self.name, args=new_args)

__str__() -> str

This function returns a string containing the PipelineSpec name, its arguments and respective values for use on the commandline.

Source code in xdsl/utils/parse_pipeline.py

def __str__(self) -> str:
    """
    This function returns a string containing the PipelineSpec name, its arguments
    and respective values for use on the commandline.
    """
    query = f"{self.name}"
    arguments_pipeline = " ".join(
        _pass_arg_list_type_str(arg_name, arg_val)
        for arg_name, arg_val in self.args.items()
    )
    query += f"{{{arguments_pipeline}}}" if self.args else ""

    return query

parse_pipeline(pipeline_spec: str) -> Iterator[PipelinePassSpec]

This takes a pipeline string and gives a representation of the specification.

Each pass is represented by a tuple of

• name: the name of the pass as string
• args: a dictionary, where each value is zero or more of (str | bool | float | int)

Source code in xdsl/utils/parse_pipeline.py

def parse_pipeline(
    pipeline_spec: str,
) -> Iterator[PipelinePassSpec]:
    """
    This takes a pipeline string and gives a representation of
    the specification.

    Each pass is represented by a tuple of:
     - name: the name of the pass as string
     - args: a dictionary, where each value is zero or more
            of (str | bool | float | int)
    """
    lexer = PipelineLexer(pipeline_spec)

    while True:
        # get the pass name
        name = lexer.lex()
        if name.kind is SpecTokenKind.EOF:
            return
        if name.kind is not SpecTokenKind.IDENT:
            raise PassPipelineParseError(name, "Expected pass name here")

        # valid next tokens are EOF, COMMA or `{`
        match lexer.lex():
            case Token(kind=SpecTokenKind.EOF):
                # EOF means we have nothing else left to parse, we are done
                yield PipelinePassSpec(name.span.text, dict())
                return
            case Token(kind=SpecTokenKind.COMMA):
                # comma means we are done parsing this pass, move on to next pass
                yield PipelinePassSpec(name.span.text, dict())
                continue
            case Token(kind=SpecTokenKind.L_BRACE):
                # `{` indicates start of args dict, so we parse that next
                yield PipelinePassSpec(name.span.text, _parse_pass_args(lexer))
            case Token(SpecTokenKind.MLIR_PIPELINE, span):
                if name.span.text != "mlir-opt":
                    raise PassPipelineParseError(
                        name,
                        "Expected `mlir-opt` to mark an MLIR pipeline here",
                    )
                yield PipelinePassSpec(
                    "mlir-opt",
                    {
                        "arguments": (
                            "--mlir-print-op-generic",
                            "--allow-unregistered-dialect",
                            "-p",
                            f"builtin.module({span.text[1:-1]})",
                        )
                    },
                )
            case invalid:
                # every other token is invalid
                raise PassPipelineParseError(
                    invalid, "Expected a comma or pass arguments here"
                )

        # check for comma or EOF
        match lexer.lex():
            case Token(kind=SpecTokenKind.EOF):
                # EOF means we are finished parsing
                return
            case Token(kind=SpecTokenKind.COMMA):
                # comma means we move on to parse the next pass spec
                continue
            case invalid:
                # every other token is invalid
                raise PassPipelineParseError(
                    invalid, "Expected a comma after pass argument dict here"
                )