Experiment

BaseModel

Bases: Protocol

Protocol for pydantic BaseModel to ensure compatibility with context

Source code in python/opsml/experiment/_experiment.pyi

class BaseModel(Protocol):
    """Protocol for pydantic BaseModel to ensure compatibility with context"""

    def model_dump(self) -> Dict[str, Any]:
        """Dump the model as a dictionary"""
        ...

    def model_dump_json(self) -> str:
        """Dump the model as a JSON string"""
        ...

    def __str__(self) -> str:
        """String representation of the model"""
        ...

__str__()

String representation of the model

Source code in python/opsml/experiment/_experiment.pyi

def __str__(self) -> str:
    """String representation of the model"""
    ...

model_dump()

Dump the model as a dictionary

Source code in python/opsml/experiment/_experiment.pyi

def model_dump(self) -> Dict[str, Any]:
    """Dump the model as a dictionary"""
    ...

model_dump_json()

Dump the model as a JSON string

Source code in python/opsml/experiment/_experiment.pyi

def model_dump_json(self) -> str:
    """Dump the model as a JSON string"""
    ...

EvalMetrics

Map of metrics used that can be used to evaluate a model. The metrics are also used when comparing a model with other models

Source code in python/opsml/experiment/_experiment.pyi

class EvalMetrics:
    """
    Map of metrics used that can be used to evaluate a model.
    The metrics are also used when comparing a model with other models
    """

    def __init__(self, metrics: Dict[str, float]) -> None:
        """
        Initialize EvalMetrics

        Args:
            metrics (Dict[str, float]):
                Dictionary of metrics containing the name of the metric as the key and its value as the value.
        """

    def __getitem__(self, key: str) -> float:
        """Get the value of a metric by name. A RuntimeError will be raised if the metric does not exist."""
        ...

__getitem__(key)

Get the value of a metric by name. A RuntimeError will be raised if the metric does not exist.

Source code in python/opsml/experiment/_experiment.pyi

def __getitem__(self, key: str) -> float:
    """Get the value of a metric by name. A RuntimeError will be raised if the metric does not exist."""
    ...

__init__(metrics)

Initialize EvalMetrics

Parameters:

Name	Type	Description	Default
metrics	Dict[str, float]	Dictionary of metrics containing the name of the metric as the key and its value as the value.	required

Source code in python/opsml/experiment/_experiment.pyi

def __init__(self, metrics: Dict[str, float]) -> None:
    """
    Initialize EvalMetrics

    Args:
        metrics (Dict[str, float]):
            Dictionary of metrics containing the name of the metric as the key and its value as the value.
    """

Experiment

Source code in python/opsml/experiment/_experiment.pyi

class Experiment:
    def start_experiment(
        self,
        space: Optional[str] = None,
        name: Optional[str] = None,
        code_dir: Optional[Path] = None,
        log_hardware: bool = False,
        experiment_uid: Optional[str] = None,
    ) -> "Experiment":
        """
        Start an Experiment

        Args:
            space (str | None):
                space to associate with `ExperimentCard`
            name (str | None):
                Name to associate with `ExperimentCard`
            code_dir (Path | None):
                Directory to log code from
            log_hardware (bool):
                Whether to log hardware information or not
            experiment_uid (str | None):
                Experiment UID. If provided, the experiment will be loaded from the server.

        Returns:
            Experiment
        """

    def __enter__(self) -> "Experiment":
        pass

    def __exit__(self, exc_type, exc_value, traceback) -> None:
        pass

    @property
    def llm(self) -> LLMEvaluator:
        """Access to LLM evaluation methods."""

    def log_metric(
        self,
        name: str,
        value: float,
        step: Optional[int] = None,
        timestamp: Optional[int] = None,
        created_at: Optional[datetime] = None,
    ) -> None:
        """
        Log a metric

        Args:
            name (str):
                Name of the metric
            value (float):
                Value of the metric
            step (int | None):
                Step of the metric
            timestamp (int | None):
                Timestamp of the metric
            created_at (datetime | None):
                Created at of the metric
        """

    def log_metrics(self, metrics: list[Metric]) -> None:
        """
        Log multiple metrics

        Args:
            metrics (list[Metric]):
                List of metrics to log
        """

    def log_eval_metrics(self, metrics: "EvalMetrics") -> None:
        """
        Log evaluation metrics

        Args:
            metrics (EvalMetrics):
                Evaluation metrics to log
        """

    def log_parameter(
        self,
        name: str,
        value: Union[int, float, str],
    ) -> None:
        """
        Log a parameter

        Args:
            name (str):
                Name of the parameter
            value (int | float | str):
                Value of the parameter
        """

    def log_parameters(self, parameters: list[Parameter] | Dict[str, Union[int, float, str]]) -> None:
        """
        Log multiple parameters

        Args:
            parameters (list[Parameter] | Dict[str, Union[int, float, str]]):
                Parameters to log
        """

    def log_artifact(
        self,
        lpath: Path,
        rpath: Optional[str] = None,
    ) -> None:
        """
        Log an artifact

        Args:
            lpath (Path):
                The local path where the artifact has been saved to
            rpath (Optional[str]):
                The path to associate with the artifact in the experiment artifact directory
                {experiment_path}/artifacts. If not provided, defaults to
                {experiment}/artifacts/{filename}
        """

    def log_figure_from_path(
        self,
        lpath: Path,
        rpath: Optional[str] = None,
    ) -> None:
        """
        Log a figure

        Args:
            lpath (Path):
                The local path where the figure has been saved to. Must be an image type
                (e.g. jpeg, tiff, png, etc.)
            rpath (Optional[str]):
                The path to associate with the figure in the experiment artifact directory
                {experiment_path}/artifacts/figures. If not provided, defaults to
                {experiment}/artifacts/figures/{filename}

        """

    def log_figure(self, name: str, figure: Any, kwargs: Optional[Dict[str, Any]] = None) -> None:
        """
        Log a figure. This method will log a matplotlib Figure object to the experiment artifacts.

        Args:
            name (str):
                Name of the figure including its file extension
            figure (Any):
                Figure to log
            kwargs (Optional[Dict[str, Any]]):
                Additional keyword arguments
        """

    def log_artifacts(
        self,
        paths: Path,
    ) -> None:
        """
        Log multiple artifacts

        Args:
            paths (Path):
                Paths to a directory containing artifacts.
                All files in the directory will be logged.
        """

    @property
    def card(self) -> "ExperimentCard":
        """
        ExperimentCard associated with the Experiment
        """

    def register_card(
        self,
        card: Union[DataCard, ModelCard, PromptCard],
        version_type: VersionType = VersionType.Minor,
        pre_tag: Optional[str] = None,
        build_tag: Optional[str] = None,
        save_kwargs: Optional[ModelSaveKwargs | DataSaveKwargs] = None,
    ) -> None:
        """Register a Card as part of an experiment

        Args:
            card (DataCard | ModelCard):
                Card to register. Can be a DataCard or a ModelCard
            version_type (VersionType):
                How to increment the version SemVer. Default is VersionType.Minor.
            pre_tag (str):
                Optional pre tag to associate with the version.
            build_tag (str):
                Optional build_tag to associate with the version.
            save_kwargs (SaveKwargs):
                Optional SaveKwargs to pass to the Card interface (If using DataCards
                and ModelCards).

        """

card property

ExperimentCard associated with the Experiment

llm property

Access to LLM evaluation methods.

log_artifact(lpath, rpath=None)

Log an artifact

Parameters:

Name	Type	Description	Default
lpath	Path	The local path where the artifact has been saved to	required
rpath	Optional[str]	The path to associate with the artifact in the experiment artifact directory {experiment_path}/artifacts. If not provided, defaults to {experiment}/artifacts/{filename}	None

Source code in python/opsml/experiment/_experiment.pyi

def log_artifact(
    self,
    lpath: Path,
    rpath: Optional[str] = None,
) -> None:
    """
    Log an artifact

    Args:
        lpath (Path):
            The local path where the artifact has been saved to
        rpath (Optional[str]):
            The path to associate with the artifact in the experiment artifact directory
            {experiment_path}/artifacts. If not provided, defaults to
            {experiment}/artifacts/{filename}
    """

log_artifacts(paths)

Log multiple artifacts

Parameters:

Name	Type	Description	Default
paths	Path	Paths to a directory containing artifacts. All files in the directory will be logged.	required

Source code in python/opsml/experiment/_experiment.pyi

def log_artifacts(
    self,
    paths: Path,
) -> None:
    """
    Log multiple artifacts

    Args:
        paths (Path):
            Paths to a directory containing artifacts.
            All files in the directory will be logged.
    """

log_eval_metrics(metrics)

Log evaluation metrics

Parameters:

Name	Type	Description	Default
metrics	EvalMetrics	Evaluation metrics to log	required

Source code in python/opsml/experiment/_experiment.pyi

def log_eval_metrics(self, metrics: "EvalMetrics") -> None:
    """
    Log evaluation metrics

    Args:
        metrics (EvalMetrics):
            Evaluation metrics to log
    """

log_figure(name, figure, kwargs=None)

Log a figure. This method will log a matplotlib Figure object to the experiment artifacts.

Parameters:

Name	Type	Description	Default
name	str	Name of the figure including its file extension	required
figure	Any	Figure to log	required
kwargs	Optional[Dict[str, Any]]	Additional keyword arguments	None

Source code in python/opsml/experiment/_experiment.pyi

def log_figure(self, name: str, figure: Any, kwargs: Optional[Dict[str, Any]] = None) -> None:
    """
    Log a figure. This method will log a matplotlib Figure object to the experiment artifacts.

    Args:
        name (str):
            Name of the figure including its file extension
        figure (Any):
            Figure to log
        kwargs (Optional[Dict[str, Any]]):
            Additional keyword arguments
    """

log_figure_from_path(lpath, rpath=None)

Log a figure

Parameters:

Name	Type	Description	Default
lpath	Path	The local path where the figure has been saved to. Must be an image type (e.g. jpeg, tiff, png, etc.)	required
rpath	Optional[str]	The path to associate with the figure in the experiment artifact directory {experiment_path}/artifacts/figures. If not provided, defaults to {experiment}/artifacts/figures/{filename}	None

Source code in python/opsml/experiment/_experiment.pyi

def log_figure_from_path(
    self,
    lpath: Path,
    rpath: Optional[str] = None,
) -> None:
    """
    Log a figure

    Args:
        lpath (Path):
            The local path where the figure has been saved to. Must be an image type
            (e.g. jpeg, tiff, png, etc.)
        rpath (Optional[str]):
            The path to associate with the figure in the experiment artifact directory
            {experiment_path}/artifacts/figures. If not provided, defaults to
            {experiment}/artifacts/figures/{filename}

    """

log_metric(name, value, step=None, timestamp=None, created_at=None)

Log a metric

Parameters:

Name	Type	Description	Default
name	str	Name of the metric	required
value	float	Value of the metric	required
step	int | None	Step of the metric	None
timestamp	int | None	Timestamp of the metric	None
created_at	datetime | None	Created at of the metric	None

Source code in python/opsml/experiment/_experiment.pyi

def log_metric(
    self,
    name: str,
    value: float,
    step: Optional[int] = None,
    timestamp: Optional[int] = None,
    created_at: Optional[datetime] = None,
) -> None:
    """
    Log a metric

    Args:
        name (str):
            Name of the metric
        value (float):
            Value of the metric
        step (int | None):
            Step of the metric
        timestamp (int | None):
            Timestamp of the metric
        created_at (datetime | None):
            Created at of the metric
    """

log_metrics(metrics)

Log multiple metrics

Parameters:

Name	Type	Description	Default
metrics	list[Metric]	List of metrics to log	required

Source code in python/opsml/experiment/_experiment.pyi

def log_metrics(self, metrics: list[Metric]) -> None:
    """
    Log multiple metrics

    Args:
        metrics (list[Metric]):
            List of metrics to log
    """

log_parameter(name, value)

Log a parameter

Parameters:

Name	Type	Description	Default
name	str	Name of the parameter	required
value	int | float | str	Value of the parameter	required

Source code in python/opsml/experiment/_experiment.pyi

def log_parameter(
    self,
    name: str,
    value: Union[int, float, str],
) -> None:
    """
    Log a parameter

    Args:
        name (str):
            Name of the parameter
        value (int | float | str):
            Value of the parameter
    """

log_parameters(parameters)

Log multiple parameters

Parameters:

Name	Type	Description	Default
parameters	list[Parameter] | Dict[str, Union[int, float, str]]	Parameters to log	required

Source code in python/opsml/experiment/_experiment.pyi

def log_parameters(self, parameters: list[Parameter] | Dict[str, Union[int, float, str]]) -> None:
    """
    Log multiple parameters

    Args:
        parameters (list[Parameter] | Dict[str, Union[int, float, str]]):
            Parameters to log
    """

register_card(card, version_type=VersionType.Minor, pre_tag=None, build_tag=None, save_kwargs=None)

Register a Card as part of an experiment

Parameters:

Name	Type	Description	Default
card	DataCard | ModelCard	Card to register. Can be a DataCard or a ModelCard	required
version_type	VersionType	How to increment the version SemVer. Default is VersionType.Minor.	Minor
pre_tag	str	Optional pre tag to associate with the version.	None
build_tag	str	Optional build_tag to associate with the version.	None
save_kwargs	SaveKwargs	Optional SaveKwargs to pass to the Card interface (If using DataCards and ModelCards).	None

Source code in python/opsml/experiment/_experiment.pyi

def register_card(
    self,
    card: Union[DataCard, ModelCard, PromptCard],
    version_type: VersionType = VersionType.Minor,
    pre_tag: Optional[str] = None,
    build_tag: Optional[str] = None,
    save_kwargs: Optional[ModelSaveKwargs | DataSaveKwargs] = None,
) -> None:
    """Register a Card as part of an experiment

    Args:
        card (DataCard | ModelCard):
            Card to register. Can be a DataCard or a ModelCard
        version_type (VersionType):
            How to increment the version SemVer. Default is VersionType.Minor.
        pre_tag (str):
            Optional pre tag to associate with the version.
        build_tag (str):
            Optional build_tag to associate with the version.
        save_kwargs (SaveKwargs):
            Optional SaveKwargs to pass to the Card interface (If using DataCards
            and ModelCards).

    """

start_experiment(space=None, name=None, code_dir=None, log_hardware=False, experiment_uid=None)

Start an Experiment

Parameters:

Name	Type	Description	Default
space	str | None	space to associate with ExperimentCard	None
name	str | None	Name to associate with ExperimentCard	None
code_dir	Path | None	Directory to log code from	None
log_hardware	bool	Whether to log hardware information or not	False
experiment_uid	str | None	Experiment UID. If provided, the experiment will be loaded from the server.	None

Returns:

Type	Description
Experiment	Experiment

Source code in python/opsml/experiment/_experiment.pyi

def start_experiment(
    self,
    space: Optional[str] = None,
    name: Optional[str] = None,
    code_dir: Optional[Path] = None,
    log_hardware: bool = False,
    experiment_uid: Optional[str] = None,
) -> "Experiment":
    """
    Start an Experiment

    Args:
        space (str | None):
            space to associate with `ExperimentCard`
        name (str | None):
            Name to associate with `ExperimentCard`
        code_dir (Path | None):
            Directory to log code from
        log_hardware (bool):
            Whether to log hardware information or not
        experiment_uid (str | None):
            Experiment UID. If provided, the experiment will be loaded from the server.

    Returns:
        Experiment
    """

LLMEvaluator

Source code in python/opsml/experiment/_experiment.pyi

class LLMEvaluator:
    @staticmethod
    def evaluate(
        records: List[LLMEvalRecord],
        metrics: List[LLMEvalMetric],
        config: Optional[EvaluationConfig] = None,
    ) -> LLMEvalResults:
        """
            Evaluate LLM responses using the provided evaluation metrics.

        Args:
            records (List[LLMEvalRecord]):
                List of LLM evaluation records to evaluate.
            metrics (List[LLMEvalMetric]):
                List of LLMEvalMetric instances to use for evaluation.
            config (Optional[EvaluationConfig]):
                Optional EvaluationConfig instance to configure evaluation options.

        Returns:
            LLMEvalResults
        """

evaluate(records, metrics, config=None) staticmethod

Evaluate LLM responses using the provided evaluation metrics.

Parameters:

Name	Type	Description	Default
records	List[LLMEvalRecord]	List of LLM evaluation records to evaluate.	required
metrics	List[LLMEvalMetric]	List of LLMEvalMetric instances to use for evaluation.	required
config	Optional[EvaluationConfig]	Optional EvaluationConfig instance to configure evaluation options.	None

Returns:

Type	Description
LLMEvalResults	LLMEvalResults

Source code in python/opsml/experiment/_experiment.pyi

@staticmethod
def evaluate(
    records: List[LLMEvalRecord],
    metrics: List[LLMEvalMetric],
    config: Optional[EvaluationConfig] = None,
) -> LLMEvalResults:
    """
        Evaluate LLM responses using the provided evaluation metrics.

    Args:
        records (List[LLMEvalRecord]):
            List of LLM evaluation records to evaluate.
        metrics (List[LLMEvalMetric]):
            List of LLMEvalMetric instances to use for evaluation.
        config (Optional[EvaluationConfig]):
            Optional EvaluationConfig instance to configure evaluation options.

    Returns:
        LLMEvalResults
    """

Metric

Source code in python/opsml/experiment/_experiment.pyi

class Metric:
    def __init__(
        self,
        name: str,
        value: float,
        step: Optional[int] = None,
        timestamp: Optional[int] = None,
        created_at: Optional[datetime] = None,
    ) -> None:
        """
        Initialize a Metric

        Args:
            name (str):
                Name of the metric
            value (float):
                Value of the metric
            step (int | None):
                Step of the metric
            timestamp (int | None):
                Timestamp of the metric
            created_at (datetime | None):
                Created at of the metric
        """

    @property
    def name(self) -> str:
        """
        Name of the metric
        """

    @property
    def value(self) -> float:
        """
        Value of the metric
        """

    @property
    def step(self) -> Optional[int]:
        """
        Step of the metric
        """

    @property
    def timestamp(self) -> Optional[int]:
        """
        Timestamp of the metric
        """

    @property
    def created_at(self) -> Optional[datetime]:
        """
        Created at of the metric
        """

created_at property

Created at of the metric

name property

Name of the metric

step property

Step of the metric

timestamp property

Timestamp of the metric

value property

Value of the metric

__init__(name, value, step=None, timestamp=None, created_at=None)

Initialize a Metric

Parameters:

Name	Type	Description	Default
name	str	Name of the metric	required
value	float	Value of the metric	required
step	int | None	Step of the metric	None
timestamp	int | None	Timestamp of the metric	None
created_at	datetime | None	Created at of the metric	None

Source code in python/opsml/experiment/_experiment.pyi

def __init__(
    self,
    name: str,
    value: float,
    step: Optional[int] = None,
    timestamp: Optional[int] = None,
    created_at: Optional[datetime] = None,
) -> None:
    """
    Initialize a Metric

    Args:
        name (str):
            Name of the metric
        value (float):
            Value of the metric
        step (int | None):
            Step of the metric
        timestamp (int | None):
            Timestamp of the metric
        created_at (datetime | None):
            Created at of the metric
    """

Parameter

Source code in python/opsml/experiment/_experiment.pyi

class Parameter:
    def __init__(
        self,
        name: str,
        value: Union[int, float, str],
    ) -> None:
        """
        Initialize a Parameter

        Args:
            name (str):
                Name of the parameter
            value (int | float | str):
                Value of the parameter
        """

    @property
    def name(self) -> str:
        """
        Name of the parameter
        """

    @property
    def value(self) -> Union[int, float, str]:
        """
        Value of the parameter
        """

name property

Name of the parameter

value property

Value of the parameter

__init__(name, value)

Initialize a Parameter

Parameters:

Name	Type	Description	Default
name	str	Name of the parameter	required
value	int | float | str	Value of the parameter	required

Source code in python/opsml/experiment/_experiment.pyi

def __init__(
    self,
    name: str,
    value: Union[int, float, str],
) -> None:
    """
    Initialize a Parameter

    Args:
        name (str):
            Name of the parameter
        value (int | float | str):
            Value of the parameter
    """

get_experiment_metrics(experiment_uid, names=None)

Get metrics of an experiment

Parameters:

Name	Type	Description	Default
experiment_uid	str	UID of the experiment	required
names	list[str] | None	Names of the metrics to get. If None, all metrics will be returned.	None

Returns:

Type	Description
Metrics	Metrics

Source code in python/opsml/experiment/_experiment.pyi

def get_experiment_metrics(
    experiment_uid: str,
    names: Optional[list[str]] = None,
) -> Metrics:
    """
    Get metrics of an experiment

    Args:
        experiment_uid (str):
            UID of the experiment
        names (list[str] | None):
            Names of the metrics to get. If None, all metrics will be returned.

    Returns:
        Metrics
    """

get_experiment_parameters(experiment_uid, names=None)

Get parameters of an experiment

Parameters:

Name	Type	Description	Default
experiment_uid	str	UID of the experiment	required
names	list[str] | None	Names of the parameters to get. If None, all parameters will be returned.	None

Returns:

Type	Description
Parameters	Parameters

Source code in python/opsml/experiment/_experiment.pyi

def get_experiment_parameters(
    experiment_uid: str,
    names: Optional[list[str]] = None,
) -> Parameters:
    """
    Get parameters of an experiment

    Args:
        experiment_uid (str):
            UID of the experiment
        names (list[str] | None):
            Names of the parameters to get. If None, all parameters will be returned.

    Returns:
        Parameters
    """

start_experiment(space=None, name=None, code_dir=None, log_hardware=False, experiment_uid=None)

Start an Experiment

Parameters:

Name	Type	Description	Default
space	str | None	space to associate with ExperimentCard	None
name	str | None	Name to associate with ExperimentCard	None
code_dir	Path | None	Directory to log code from	None
log_hardware	bool	Whether to log hardware information or not	False
experiment_uid	str | None	Experiment UID. If provided, the experiment will be loaded from the server.	None

Returns:

Type	Description
Experiment	Experiment

Source code in python/opsml/experiment/_experiment.pyi

def start_experiment(
    space: Optional[str] = None,
    name: Optional[str] = None,
    code_dir: Optional[Path] = None,
    log_hardware: bool = False,
    experiment_uid: Optional[str] = None,
) -> Experiment:
    """
    Start an Experiment

    Args:
        space (str | None):
            space to associate with `ExperimentCard`
        name (str | None):
            Name to associate with `ExperimentCard`
        code_dir (Path | None):
            Directory to log code from
        log_hardware (bool):
            Whether to log hardware information or not
        experiment_uid (str | None):
            Experiment UID. If provided, the experiment will be loaded from the server.

    Returns:
        Experiment
    """