Data Types#

enum AnnotationType(value)#

Member Type:: str

Valid values are as follows:

GOOD = <AnnotationType.GOOD: 'good'>#

BAD = <AnnotationType.BAD: 'bad'>#

UNKNOWN = <AnnotationType.UNKNOWN: 'unknown'>#

The Enum and its members also have the following methods:

__new__(value)#

class Application#

Attributes:

description
log_latest_insert_time_epoch
n_of_interactions
n_of_llm_properties
notifications_enabled

__init__(id: int, name: str, kind: ApplicationType, created_at: datetime, updated_at: datetime, in_progress: bool, versions: List[ApplicationVersion], interaction_types: List[str], description: str | None = None, log_latest_insert_time_epoch: int | None = None, n_of_llm_properties: int | None = None, n_of_interactions: int | None = None, notifications_enabled: bool | None = None) → None#

enum ApplicationType(value)#

Member Type:: str

Valid values are as follows:

QA = <ApplicationType.QA: 'Q&A'>#

OTHER = <ApplicationType.OTHER: 'OTHER'>#

SUMMARIZATION = <ApplicationType.SUMMARIZATION: 'SUMMARIZATION'>#

GENERATION = <ApplicationType.GENERATION: 'GENERATION'>#

CLASSIFICATION = <ApplicationType.CLASSIFICATION: 'CLASSIFICATION'>#

FEATURE_EXTRACTION = <ApplicationType.FEATURE_EXTRACTION: 'FEATURE EXTRACTION'>#

RETRIEVAL = <ApplicationType.RETRIEVAL: 'Retrieval'>#

CHAT = <ApplicationType.CHAT: 'Chat'>#

CHAIN = <ApplicationType.CHAIN: 'Chain'>#

ROOT = <ApplicationType.ROOT: 'Root'>#

LLM = <ApplicationType.LLM: 'LLM'>#

AGENT = <ApplicationType.AGENT: 'Agent'>#

TOOL = <ApplicationType.TOOL: 'Tool'>#

The Enum and its members also have the following methods:

__new__(value)#

class ApplicationVersion#

A dataclass representing an Application Version.

Attributes:

idint: Version id
namestr: Version name
ai_modelstr or None: AI model used within this version DEPRECATED: This field is deprecated and will be removed in a future version. It is no longer used by the application.
created_atdatetime: Version created at timestamp
updated_atdatetime: Version updated at timestamp
customlist of dict: Additional details about the version as key-value pairs This member is deprecated. It will be removed in future versions. Use additional_fields instead.
additional_fieldsdict: Additional details about the version as dict

__init__(id: int, name: str, created_at: datetime, updated_at: datetime, ai_model: str | None = None, description: str | None = None, additional_fields: Dict[str, Any] | None = None) → None#

class ApplicationVersionSchema#

Attributes:

additional_fields
description

Methods

to_json

__init__(name: str, description: str | None = None, additional_fields: Dict[str, Any] | None = None) → None#

enum BuiltInInteractionType(value)#

Member Type:: str

Valid values are as follows:

QA = <BuiltInInteractionType.QA: 'Q&A'>#

OTHER = <BuiltInInteractionType.OTHER: 'Other'>#

SUMMARIZATION = <BuiltInInteractionType.SUMMARIZATION: 'Summarization'>#

CLASSIFICATION = <BuiltInInteractionType.CLASSIFICATION: 'Classification'>#

GENERATION = <BuiltInInteractionType.GENERATION: 'Generation'>#

FEATURE_EXTRACTION = <BuiltInInteractionType.FEATURE_EXTRACTION: 'Feature Extraction'>#

RETRIEVAL = <BuiltInInteractionType.RETRIEVAL: 'Retrieval'>#

CHAT = <BuiltInInteractionType.CHAT: 'Chat'>#

CHAIN = <BuiltInInteractionType.CHAIN: 'Chain'>#

ROOT = <BuiltInInteractionType.ROOT: 'Root'>#

LLM = <BuiltInInteractionType.LLM: 'LLM'>#

AGENT = <BuiltInInteractionType.AGENT: 'Agent'>#

TOOL = <BuiltInInteractionType.TOOL: 'Tool'>#

The Enum and its members also have the following methods:

__new__(value)#

class CreateInteractionTypeVersionData#

A dataclass for creating interaction type version data.

Attributes:

interaction_type_idint: Interaction type id
application_version_idint: Application version id
modelstr or None: Model name
promptstr or None: Prompt template
metadata_paramsdict: Additional metadata parameters

Methods

to_json

__init__(interaction_type_id: int, application_version_id: int, model: str | None = None, prompt: str | None = None, metadata_params: Dict[str, Any] = None) → None#

class Dataset#

Data class representing a dataset.

Methods

from_dict

__init__(id: int, application_id: int, dataset_name: str, samples_count: int, created_at: datetime, updated_at: datetime) → None#

class DatasetRunResult#

Result of running a dataset on a deployment.

Attributes:

additional_headers
app_name
env_type
failed_results: Get only the failed sample results.
success_rate: Calculate success rate as a percentage.
successful_results: Get only the successful sample results.
version_name

Methods

rerun_failed([show_progress])

Re-run only the failed samples from this result and update in place.

__init__(dataset_name: str, deployment_name: str, total_samples: int, successful_samples: int, failed_samples: int, duration_seconds: float, results: List[DatasetRunSampleResult], app_name: str | None = None, version_name: str | None = None, env_type: str | None = None, additional_headers: Dict[str, str] | None = None, verify_ssl: bool = True, _deployment: Deployment | None = None, iteration: int = 1) → None#

property failed_results: List[DatasetRunSampleResult]#: Get only the failed sample results.

async rerun_failed(show_progress: bool = True) → None#

Re-run only the failed samples from this result and update in place.

This is a convenience method that re-runs failed samples using the same configuration (app_name, deployment, version, etc.) from the original run, then merges the successful retries back into this result.

Note: This is an async method and must be called with await: >>> await result.rerun_failed()

Parameters:

show_progressbool, default=True: Whether to show a live progress widget (if in notebook with ipywidgets installed)

Examples

>>> result = await client.execute_app("my-app", "test-dataset", "prod-deployment")
>>> print(f"Initial: {result.successful_samples}/{result.total_samples} succeeded")
>>> if result.failed_samples > 0:
...     await result.rerun_failed()  # Updates result in place
...     print(f"After retry: {result.successful_samples}/{result.total_samples} succeeded")
>>> result  # Display result in notebook - shows interactive widget

property success_rate: float#: Calculate success rate as a percentage.

property successful_results: List[DatasetRunSampleResult]#: Get only the successful sample results.

class DatasetRunSampleResult#

Result of running a single dataset sample.

Attributes:

deployment_response
duration_seconds
error_message
error_type
status_code

__init__(sample_id: int, sample_input: Any, sample_output: Any | None, sample_metadata: Dict[str, Any] | None, success: bool, deployment_response: Dict[str, Any] | None = None, error_message: str | None = None, error_type: str | None = None, duration_seconds: float | None = None, retries: int = 0, status_code: int | None = None) → None#

class DatasetSample#

Data class representing a dataset sample.

Attributes:

created_at
updated_at

__init__(id: int, dataset_id: int, input: Any, output: Any | None, sample_metadata: Dict[str, Any] | None, created_at: datetime | None = None, updated_at: datetime | None = None) → None#

class Deployment#

Data class representing a deployment configuration.

Methods

from_dict

__init__(id: int, application_id: int, deployment_name: str, deployment_url: str, timeout: int, max_concurrent: int, max_retries: int, created_at: datetime, updated_at: datetime, headers: List[DeploymentHeader]) → None#

class DeploymentHeader#

Data class representing a deployment header.

Methods

from_dict

__init__(id: int, deployment_id: int, name: str, value: str) → None#

enum EnvType(value)#

Member Type:: str

Valid values are as follows:

PROD = <EnvType.PROD: 'PROD'>#

EVAL = <EnvType.EVAL: 'EVAL'>#

PENTEST = <EnvType.PENTEST: 'PENTEST'>#

The Enum and its members also have the following methods:

__new__(value)#

class Interaction#

__init__(user_interaction_id: str | int, input: str, output: str, information_retrieval: str | List[str], history: str | List[str], full_prompt: str, expected_output: str, is_completed: bool, metadata: Dict[str, str], tokens: int, input_tokens: int | None, output_tokens: int | None, model: str | None, model_provider: str | None, input_cost: float | None, output_cost: float | None, cost: float | None, builtin_properties: Dict[str, Any], user_value_properties: Dict[str, Any], custom_prompt_properties: Dict[str, Any], properties_reasons: Dict[str, Any], created_at: datetime, interaction_datetime: datetime, interaction_type: str, topic: str, session_id: str | int, annotation: AnnotationType | None, annotation_reason: str | None) → None#

enum InteractionCompleteEvents(value)#

Member Type:: str

Valid values are as follows:

TOPICS_COMPLETED = <InteractionCompleteEvents.TOPICS_COMPLETED: 'topics_completed'>#

PROPERTIES_COMPLETED = <InteractionCompleteEvents.PROPERTIES_COMPLETED: 'properties_completed'>#

SIMILARITY_COMPLETED = <InteractionCompleteEvents.SIMILARITY_COMPLETED: 'similarity_completed'>#

LLM_PROPERTIES_COMPLETED = <InteractionCompleteEvents.LLM_PROPERTIES_COMPLETED: 'llm_properties_completed'>#

ANNOTATION_COMPLETED = <InteractionCompleteEvents.ANNOTATION_COMPLETED: 'annotation_completed'>#

DC_EVALUATION_COMPLETED = <InteractionCompleteEvents.DC_EVALUATION_COMPLETED: 'dc_evaluation_completed'>#

BUILTIN_LLM_PROPERTIES_COMPLETED = <InteractionCompleteEvents.BUILTIN_LLM_PROPERTIES_COMPLETED: 'builtin_llm_properties_completed'>#

The Enum and its members also have the following methods:

__new__(value)#

class InteractionType#

__init__(id: int, name: str) → None#

class InteractionTypeVersionData#

A dataclass representing interaction type version data.

Attributes:

idint: Interaction type version data id
interaction_type_idint: Interaction type id
application_version_idint: Application version id
modelstr or None: Model name
promptstr or None: Prompt template
metadata_paramsdict: Additional metadata parameters
created_atdatetime: Created at timestamp
updated_atdatetime: Updated at timestamp

__init__(id: int, interaction_type_id: int, application_version_id: int, model: str | None = None, prompt: str | None = None, metadata_params: Dict[str, Any] = None, created_at: datetime = None, updated_at: datetime = None) → None#

class LogInteraction#

A dataclass representing a new interaction object.

Attributes:

inputstr: Input data
outputstr: Output data
expected_outputstr, optional: Full expected output data, defaults to None
full_promptstr, optional: Full prompt data, defaults to None
annotationAnnotationType, optional: Annotation type of the interaction, defaults to None
user_interaction_idstr, optional: Unique identifier of the interaction, defaults to None
stepslist of Step, optional: List of steps taken during the interaction, defaults to None
user_value_propertieslist of UserValueProperty, optional: Additional user value properties, defaults to None
information_retrievalstr, optional: Information retrieval, defaults to None
historystr, optional: History (for instance “chat history”), defaults to None
annotation_reasonstr, optional: Reason for the annotation, defaults to None
started_atdatetime or float, optional: Timestamp the interaction started at. Datetime format is deprecated, use timestamp instead
finished_atdatetime or float, optional: Timestamp the interaction finished at. Datetime format is deprecated, use timestamp instead
vuln_typestr, optional: Type of vulnerability (Only used in case of EnvType.PENTEST and must be sent there), defaults to None
vuln_trigger_strstr, optional: Vulnerability trigger string (Only used in case of EnvType.PENTEST and is optional there), defaults to None
session_id: str, optional: The identifier for the session associated with this interaction. If not provided, a session ID will be automatically generated.
interaction_type: str, optional: The type of interaction. None is deprecated. If not provided, the interaction type will default to the application’s default type.
metadata: t.Dict[str, str], optional: Metdata for the interaction.
tokens: int, optional: Token count for the interaction.

Methods

to_json

__init__(input: str | None = None, output: str | None = None, information_retrieval: str | List[str] | None = None, history: str | List[str] | None = None, full_prompt: str | None = None, expected_output: str | None = None, is_completed: bool = True, metadata: Dict[str, str] | None = None, tokens: int | None = None, input_tokens: int | None = None, output_tokens: int | None = None, model: str | None = None, model_provider: str | None = None, annotation: AnnotationType | str | None = None, annotation_reason: str | None = None, steps: List[Step] | None = None, user_value_properties: List[UserValueProperty] | None = None, started_at: datetime | float | None = None, finished_at: datetime | float | None = None, user_interaction_id: str | int | None = None, vuln_type: str | None = None, vuln_trigger_str: str | None = None, topic: str | None = None, interaction_type: str | None = None, session_id: str | int | None = None) → None#

enum PropertyColumnType(value)#

Member Type:: str

Valid values are as follows:

CATEGORICAL = <PropertyColumnType.CATEGORICAL: 'categorical'>#

NUMERIC = <PropertyColumnType.NUMERIC: 'numeric'>#

The Enum and its members also have the following methods:

__new__(value)#

class Span#

A dataclass representing a span within a trace for tracking nested operations.

A Span represents a unit of work within a distributed trace, allowing you to track hierarchical relationships between operations. This is designed to work with the OtelParser system for converting spans into interactions.

Attributes:

span_idstr: The unique identifier for this span
span_namestr: The name of this span, describing the operation being tracked
span_kindSpanKind: The type of span (Root is CHAIN without parent_id)
trace_idstr: The unique identifier for the trace this span belongs to. All spans in the same trace share this ID
parent_idstr or None: The ID of the parent span for tree structure. None if this is the root span
started_atfloat: Timestamp when the span started (numeric value for ordering)
finished_atfloat: Timestamp when the span finished (numeric value for ordering)
status_code{‘OK’, ‘ERROR’}, optional: The status code indicating whether the span completed successfully. ‘OK’ indicates success, ‘ERROR’ indicates failure. Defaults to ‘OK’
status_descriptionstr or None, optional: Human-readable description providing additional context about the status, particularly useful for explaining error conditions. Defaults to None
inputstr, optional: Input data for this span’s operation, defaults to None
outputstr, optional: Output data from this span’s operation, defaults to None
full_promptstr, optional: Full prompt data, defaults to None
expected_outputstr, optional: Expected output data, defaults to None
information_retrievalstr, optional: Information retrieval data, defaults to None
tokensint, optional: Token count for aggregation purposes, defaults to None
input_tokensint, optional: Input token count for the span, defaults to None
output_tokensint, optional: Output token count for the span, defaults to None
modelstr, optional: The model name used for this span (e.g., ‘gpt-4’), used for cost calculation, defaults to None
model_providerstr, optional: The model provider used for this span (e.g., ‘openai’), used for cost calculation, defaults to None
graph_parent_namestr, optional: Graph metadata indicating “who triggered me” (the span_name of another span), defaults to None
session_idstr, optional: The identifier for the session associated with this span, defaults to None
metadatat.Dict[str, str], optional: Additional metadata for the span, defaults to None
eventslist of SpanEvent, optional: List of events that occurred during this span’s execution, defaults to None
user_value_propertieslist of UserValueProperty, optional: User-provided custom properties for the span, defaults to None

Methods

to_span_data

__init__(span_id: str, span_name: str, trace_id: str, span_kind: SpanKind, parent_id: str | None, started_at: float, finished_at: float, status_code: Literal['OK', 'ERROR'] = 'OK', status_description: str | None = None, input: str | None = None, output: str | None = None, full_prompt: str | None = None, expected_output: str | None = None, information_retrieval: List[str] | None = None, tokens: int | None = None, input_tokens: int | None = None, output_tokens: int | None = None, model: str | None = None, model_provider: str | None = None, graph_parent_name: str | None = None, session_id: str | None = None, metadata: Dict[str, str] | None = None, events: List[SpanEvent] | None = None, user_value_properties: List[UserValueProperty] | None = None) → None#

class SpanEvent#

A dataclass representing an event that occurred during a span’s execution.

Attributes:

namestr: The name of the event
timestampfloat: The timestamp when the event occurred (Unix timestamp)
attributesdict, optional: Additional attributes associated with the event

__init__(name: str, timestamp: float, attributes: Dict[str, Any] | None = None) → None#

enum SpanKind(value)#

Member Type:: str

Valid values are as follows:

LLM = <SpanKind.LLM: 'LLM'>#

TOOL = <SpanKind.TOOL: 'TOOL'>#

CHAIN = <SpanKind.CHAIN: 'CHAIN'>#

AGENT = <SpanKind.AGENT: 'AGENT'>#

RETRIEVAL = <SpanKind.RETRIEVAL: 'RETRIEVER'>#

The Enum and its members also have the following methods:

__new__(value)#

class Step#

Methods

as_jsonl
to_json

__init__(name: str, value: str) → None#

class UpdateInteractionTypeVersionData#

A dataclass for updating interaction type version data.

Attributes:

modelstr or None: Model name
promptstr or None: Prompt template
metadata_paramsdict or None: Additional metadata parameters

Methods

to_json

__init__(model: str | None = None, prompt: str | None = None, metadata_params: Dict[str, Any] | None = None) → None#

class UserValueProperty#

Data class representing user provided property

Attributes:

reason

__init__(name: str, value: Any, reason: str | None = None) → None#

class UserValuePropertyType#

Attributes:

description

__init__(display_name: str, type: PropertyColumnType | str, description: str | None = None) → None#