Arg spec

arg_spec

When passing command-line arguments to xdsl-opt, it may be useful to parametrize them. The parametrized argument model object ArgSpec holds the name of the argument, and a mapping from a key to a tuple of parameters, described below.

This is used when building pass pipelines.

ParameterType = str | int | bool | float module-attribute

The only types that can be used as ArgSpec parameters.

ParameterListType = tuple[ParameterType, ...] module-attribute

The ArgSpec holds a dictionary from strings to lists of parameters.

SpecToken: TypeAlias = Token[SpecTokenKind] module-attribute

ArgSpec dataclass

A specification for a command-line argument name and its parameters.

Source code in xdsl/utils/arg_spec.py

@dataclass(eq=True, frozen=True)
class ArgSpec:
    """
    A specification for a command-line argument name and its parameters.
    """

    name: str
    """
    The name of the argument.
    """
    parameters: dict[str, ParameterListType]
    """
    The parameters of this argument.
    """

    def normalize_parameter_names(self) -> ArgSpec:
        """
        This normalized all arg names by replacing `-` with `_`
        """
        new_args: dict[str, ParameterListType] = dict()
        for k, v in self.parameters.items():
            new_args[k.replace("-", "_")] = v
        return ArgSpec(name=self.name, parameters=new_args)

    @staticmethod
    def _spec_parameter_type_str(arg: ParameterType) -> str:
        match arg:
            case bool():
                return str(arg).lower()
            case str():
                return f'"{arg}"'
            case int():
                return str(arg)
            case float():
                return str(arg)

    @staticmethod
    def _spec_parameter_list_type_str(name: str, arg: ParameterListType) -> str:
        if arg:
            return f"{name}={','.join(ArgSpec._spec_parameter_type_str(val) for val in arg)}"
        else:
            return name

    def __str__(self) -> str:
        """
        This function returns a string containing the PipelineSpec name, its arguments
        and respective values for use on the commandline.
        """
        query = f"{self.name}"
        arguments_pipeline = " ".join(
            ArgSpec._spec_parameter_list_type_str(arg_name, arg_val)
            for arg_name, arg_val in self.parameters.items()
        )
        query += f"{{{arguments_pipeline}}}" if self.parameters else ""

        return query

name: str instance-attribute

The name of the argument.

parameters: dict[str, ParameterListType] instance-attribute

The parameters of this argument.

__init__(name: str, parameters: dict[str, ParameterListType]) -> None

normalize_parameter_names() -> ArgSpec

This normalized all arg names by replacing - with _

Source code in xdsl/utils/arg_spec.py

def normalize_parameter_names(self) -> ArgSpec:
    """
    This normalized all arg names by replacing `-` with `_`
    """
    new_args: dict[str, ParameterListType] = dict()
    for k, v in self.parameters.items():
        new_args[k.replace("-", "_")] = v
    return ArgSpec(name=self.name, parameters=new_args)

__str__() -> str

This function returns a string containing the PipelineSpec name, its arguments and respective values for use on the commandline.

Source code in xdsl/utils/arg_spec.py

def __str__(self) -> str:
    """
    This function returns a string containing the PipelineSpec name, its arguments
    and respective values for use on the commandline.
    """
    query = f"{self.name}"
    arguments_pipeline = " ".join(
        ArgSpec._spec_parameter_list_type_str(arg_name, arg_val)
        for arg_name, arg_val in self.parameters.items()
    )
    query += f"{{{arguments_pipeline}}}" if self.parameters else ""

    return query

ArgSpecConvertible dataclass

Bases: ABC

A base class for frozen dataclasses with a name: ClassVar[str] that can be instantiated from an ArgSpec and serialized back to one.

Subclasses must be decorated with @dataclass(frozen=True).

Only the following types are supported as argument types:

Base types: int | float | bool | string N-tuples of base types: tuple[int, ...], tuple[int|float, ...], tuple[int, ...] | tuple[float, ...] Top-level optional: ... | None

Arguments are formatted as follows::

Spec arg                            Mapped to class field
-------------------------           ------------------------------
my-thing{arg-1=1}                   arg_1: int             = 1
my-thing{arg-1}                     arg_1: int | None      = None
my-thing{arg-1=1,2,3}              arg_1: tuple[int, ...] = (1, 2, 3)
my-thing{arg-1=true}               arg_1: bool | None     = True

Source code in xdsl/utils/arg_spec.py

@dataclass(frozen=True)
class ArgSpecConvertible(ABC):
    """
    A base class for frozen dataclasses with a ``name: ClassVar[str]`` that can
    be instantiated from an ``ArgSpec`` and serialized back to one.

    Subclasses must be decorated with ``@dataclass(frozen=True)``.

    Only the following types are supported as argument types:

    Base types:                int | float | bool | string
    N-tuples of base types:
        tuple[int, ...], tuple[int|float, ...], tuple[int, ...] | tuple[float, ...]
    Top-level optional:        ... | None

    Arguments are formatted as follows::

        Spec arg                            Mapped to class field
        -------------------------           ------------------------------
        my-thing{arg-1=1}                   arg_1: int             = 1
        my-thing{arg-1}                     arg_1: int | None      = None
        my-thing{arg-1=1,2,3}              arg_1: tuple[int, ...] = (1, 2, 3)
        my-thing{arg-1=true}               arg_1: bool | None     = True
    """

    name: ClassVar[str]

    @classmethod
    def from_spec(cls, spec: ArgSpec) -> Self:
        """
        Takes an ArgSpec, does type checking on the arguments, and instantiates
        an instance of this class from the spec.
        """
        if spec.name != cls.name:
            raise ValueError(
                f"Spec name mismatch: got '{spec.name}', expected '{cls.name}'."
            )

        spec_arguments_dict: dict[str, ParameterListType] = (
            spec.normalize_parameter_names().parameters
        )

        fields: tuple[Field[Any], ...] = dataclasses.fields(cls)

        arg_dict = dict[str, ParameterListType | ParameterType | None]()

        required = cls.required_fields()

        field_types = get_type_hints(cls)

        for op_field in fields:
            if op_field.name == "name" or not op_field.init:
                continue
            if op_field.name not in spec_arguments_dict:
                if op_field.name not in required:
                    arg_dict[op_field.name] = _get_default(op_field)
                    continue
                raise ValueError(f'{cls.name} requires argument "{op_field.name}"')

            field_type = field_types[op_field.name]
            arg_dict[op_field.name] = _convert_arg_to_type(
                spec_arguments_dict.pop(op_field.name),
                field_type,
            )

        if len(spec_arguments_dict) != 0:
            arguments_str = ", ".join(f'"{arg}"' for arg in spec_arguments_dict)
            fields_str = ", ".join(f'"{field.name}"' for field in fields)
            raise ValueError(
                f"Provided arguments [{arguments_str}] not found in expected "
                f"arguments [{fields_str}]"
            )

        return cls(**arg_dict)

    @classmethod
    def required_fields(cls) -> set[str]:
        """
        Inspects the definition for fields that do not have default values.
        """
        return {
            field.name for field in dataclasses.fields(cls) if not _is_optional(field)
        }

    def spec(self, *, include_default: bool = False) -> ArgSpec:
        """
        Returns an ArgSpec representation of this instance.

        If ``include_default`` is ``True``, then optional arguments with default
        values are also included in the spec.
        """
        fields = dataclasses.fields(self)
        args: dict[str, ParameterListType] = {}

        for op_field in fields:
            name = op_field.name
            if name == "name" or not op_field.init:
                continue

            val = getattr(self, name)

            if _is_optional(op_field):
                if val == _get_default(op_field) and not include_default:
                    continue

            if val is None:
                arg_list = ()
            elif isinstance(val, ParameterType):
                arg_list = (val,)
            else:
                arg_list = val

            args[name] = arg_list
        return ArgSpec(self.name, args)

    def __str__(self) -> str:
        return str(self.spec())

name: str class-attribute

__init__() -> None

from_spec(spec: ArgSpec) -> Self classmethod

Takes an ArgSpec, does type checking on the arguments, and instantiates an instance of this class from the spec.

Source code in xdsl/utils/arg_spec.py

@classmethod
def from_spec(cls, spec: ArgSpec) -> Self:
    """
    Takes an ArgSpec, does type checking on the arguments, and instantiates
    an instance of this class from the spec.
    """
    if spec.name != cls.name:
        raise ValueError(
            f"Spec name mismatch: got '{spec.name}', expected '{cls.name}'."
        )

    spec_arguments_dict: dict[str, ParameterListType] = (
        spec.normalize_parameter_names().parameters
    )

    fields: tuple[Field[Any], ...] = dataclasses.fields(cls)

    arg_dict = dict[str, ParameterListType | ParameterType | None]()

    required = cls.required_fields()

    field_types = get_type_hints(cls)

    for op_field in fields:
        if op_field.name == "name" or not op_field.init:
            continue
        if op_field.name not in spec_arguments_dict:
            if op_field.name not in required:
                arg_dict[op_field.name] = _get_default(op_field)
                continue
            raise ValueError(f'{cls.name} requires argument "{op_field.name}"')

        field_type = field_types[op_field.name]
        arg_dict[op_field.name] = _convert_arg_to_type(
            spec_arguments_dict.pop(op_field.name),
            field_type,
        )

    if len(spec_arguments_dict) != 0:
        arguments_str = ", ".join(f'"{arg}"' for arg in spec_arguments_dict)
        fields_str = ", ".join(f'"{field.name}"' for field in fields)
        raise ValueError(
            f"Provided arguments [{arguments_str}] not found in expected "
            f"arguments [{fields_str}]"
        )

    return cls(**arg_dict)

required_fields() -> set[str] classmethod

Inspects the definition for fields that do not have default values.

Source code in xdsl/utils/arg_spec.py

@classmethod
def required_fields(cls) -> set[str]:
    """
    Inspects the definition for fields that do not have default values.
    """
    return {
        field.name for field in dataclasses.fields(cls) if not _is_optional(field)
    }

spec(*, include_default: bool = False) -> ArgSpec

Returns an ArgSpec representation of this instance.

If include_default is True, then optional arguments with default values are also included in the spec.

Source code in xdsl/utils/arg_spec.py

def spec(self, *, include_default: bool = False) -> ArgSpec:
    """
    Returns an ArgSpec representation of this instance.

    If ``include_default`` is ``True``, then optional arguments with default
    values are also included in the spec.
    """
    fields = dataclasses.fields(self)
    args: dict[str, ParameterListType] = {}

    for op_field in fields:
        name = op_field.name
        if name == "name" or not op_field.init:
            continue

        val = getattr(self, name)

        if _is_optional(op_field):
            if val == _get_default(op_field) and not include_default:
                continue

        if val is None:
            arg_list = ()
        elif isinstance(val, ParameterType):
            arg_list = (val,)
        else:
            arg_list = val

        args[name] = arg_list
    return ArgSpec(self.name, args)

__str__() -> str

Source code in xdsl/utils/arg_spec.py

def __str__(self) -> str:
    return str(self.spec())

SpecTokenKind

Bases: Enum

Source code in xdsl/utils/arg_spec.py

class SpecTokenKind(Enum):
    EOF = object()

    IDENT = object()
    L_BRACE = "{"
    R_BRACE = "}"
    EQUALS = "="
    NUMBER = object()
    SPACE = object()
    STRING_LIT = object()
    MLIR_PIPELINE = object()
    COMMA = ","

EOF = object() class-attribute instance-attribute

IDENT = object() class-attribute instance-attribute

L_BRACE = '{' class-attribute instance-attribute

R_BRACE = '}' class-attribute instance-attribute

EQUALS = '=' class-attribute instance-attribute

NUMBER = object() class-attribute instance-attribute

SPACE = object() class-attribute instance-attribute

STRING_LIT = object() class-attribute instance-attribute

MLIR_PIPELINE = object() class-attribute instance-attribute

COMMA = ',' class-attribute instance-attribute

PipelineLexer

This tokenizes a pass declaration string: pipeline ::= pipeline-element (, pipeline-element) pipeline-element ::= MLIR_PIPELINE | pass-name options? options ::= { options-element ( options-element) } options-element ::= key (= value (, value)* )?

key ::= IDENT pass-name ::= IDENT value ::= NUMBER | BOOL | IDENT | STRING_LITERAL

Source code in xdsl/utils/arg_spec.py

class PipelineLexer:
    """
    This tokenizes a pass declaration string:
    pipeline          ::= pipeline-element (`,` pipeline-element)*
    pipeline-element  ::= MLIR_PIPELINE
                        | pass-name options?
    options           ::= `{` options-element ( ` ` options-element)* `}`
    options-element   ::= key (`=` value (`,` value)* )?

    key       ::= IDENT
    pass-name ::= IDENT
    value     ::= NUMBER | BOOL | IDENT | STRING_LITERAL
    """

    _stream: Iterator[SpecToken]
    _peeked: SpecToken | None

    def __init__(self, input_str: str):
        self._stream = PipelineLexer._generator(input_str)
        self._peeked = None

    @staticmethod
    def _generator(input_str: str) -> Iterator[SpecToken]:
        input = Input(input_str, "pass-pipeline")
        pos = 0
        end = len(input_str)

        if len(input_str) == 0:
            yield SpecToken(SpecTokenKind.EOF, Span(pos, pos + 1, input))
            return

        while True:
            token: SpecToken | None = None
            for pattern, kind in _lexer_rules:
                if (match := pattern.match(input_str, pos)) is not None:
                    token = SpecToken(kind, Span(match.start(), match.end(), input))
                    pos = match.end()
                    break
            if token is None:
                raise ArgSpecParseError(
                    SpecToken(SpecTokenKind.IDENT, Span(pos, pos + 1, input)),
                    "Unknown token",
                )
            yield token
            if pos >= end:
                yield SpecToken(SpecTokenKind.EOF, Span(pos, pos + 1, input))
                return

    def lex(self) -> SpecToken:
        token = self.peek()
        self._peeked = None
        return token

    def peek(self) -> SpecToken:
        if self._peeked is None:
            self._peeked = next(self._stream)
        return self._peeked

__init__(input_str: str)

Source code in xdsl/utils/arg_spec.py

def __init__(self, input_str: str):
    self._stream = PipelineLexer._generator(input_str)
    self._peeked = None

lex() -> SpecToken

Source code in xdsl/utils/arg_spec.py

def lex(self) -> SpecToken:
    token = self.peek()
    self._peeked = None
    return token

peek() -> SpecToken

Source code in xdsl/utils/arg_spec.py

def peek(self) -> SpecToken:
    if self._peeked is None:
        self._peeked = next(self._stream)
    return self._peeked

parse_pipeline(pipeline_spec: str) -> Iterator[ArgSpec]

This takes a pipeline string and gives a representation of the specification.

Each pass is represented by a tuple of

• name: the name of the pass as string
• args: a dictionary, where each value is zero or more of (str | bool | float | int)

Source code in xdsl/utils/arg_spec.py

def parse_pipeline(
    pipeline_spec: str,
) -> Iterator[ArgSpec]:
    """
    This takes a pipeline string and gives a representation of
    the specification.

    Each pass is represented by a tuple of:
     - name: the name of the pass as string
     - args: a dictionary, where each value is zero or more
            of (str | bool | float | int)
    """
    lexer = PipelineLexer(pipeline_spec)

    while True:
        if lexer.peek().kind is SpecTokenKind.EOF:
            return

        yield _parse_spec(lexer)

        # check for comma or EOF
        match lexer.lex():
            case Token(kind=SpecTokenKind.EOF):
                # EOF means we are finished parsing
                return
            case Token(kind=SpecTokenKind.COMMA):
                # comma means we move on to parse the next pass spec
                continue
            case invalid:
                # every other token is invalid
                raise ArgSpecParseError(
                    invalid, "Expected a comma after pass argument dict here"
                )

parse_spec(spec: str) -> ArgSpec

Parses a spec, with optional arguments, or raises a ArgSpecParseError if one cannot be parsed.

Source code in xdsl/utils/arg_spec.py

def parse_spec(spec: str) -> ArgSpec:
    """
    Parses a spec, with optional arguments, or raises a `ArgSpecParseError` if one
    cannot be parsed.
    """
    lexer = PipelineLexer(spec)
    return _parse_spec(lexer)