Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ switchyard-protocol = { path = "crates/protocol", version = "0.3.0" }
switchyard-runner = { path = "crates/switchyard-runner", version = "0.3.0" }
switchyard-server = { path = "crates/switchyard-server", version = "0.3.0" }
switchyard-translation = { path = "crates/switchyard-translation", version = "0.3.0" }
strum = "0.28"
strum_macros = "0.28"
thiserror = "2"
tokio = { version = "1", features = ["full"] }
Expand Down
1 change: 1 addition & 0 deletions crates/switchyard-runner/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ serde.workspace = true
serde_json.workspace = true
switchyard-llm-client.workspace = true
switchyard-protocol.workspace = true
strum.workspace = true
strum_macros.workspace = true
thiserror.workspace = true
toml = "1.1"
Expand Down
5 changes: 5 additions & 0 deletions crates/switchyard-runner/src/algorithm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,11 @@ impl StageClassifierConfig {
}

impl AlgorithmSpec {
/// Whether the algorithm can be built independently for two target maps.
pub(crate) const fn supports_privacy_lanes(&self) -> bool {
!matches!(self, Self::PrefillRouter { .. })
}

pub(crate) fn decision_judge(&self) -> Option<(&str, &DecisionJudgeRouteConfig)> {
match self {
Self::LlmClassifier { config, .. } => config
Expand Down
123 changes: 113 additions & 10 deletions crates/switchyard-runner/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use switchyard_llm_client::{
};
use switchyard_protocol::{Category, ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat};

use crate::privacy::PrivacyPolicy;
use crate::route::ExecutionLane;
use crate::{
AlgorithmSpec, AuxiliaryTarget, CallerAuthKind, DecisionTarget, ModelCapabilities, Route,
Expand Down Expand Up @@ -75,9 +76,18 @@ struct RouteConfig {
tool_calling: Option<bool>,
reasoning: Option<bool>,
vision: Option<bool>,
privacy: Option<PrivacyConfig>,
algorithm: AlgorithmSpec,
}

#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct PrivacyConfig {
restricted_targets: BTreeMap<String, String>,
#[serde(default)]
accept_external_signal: bool,
}

struct TargetPromptPolicy {
prompts: HashMap<ModelId, String>,
routing_answer_target: Option<ModelId>,
Expand All @@ -97,6 +107,11 @@ struct ResolvedDecisionTarget<'a> {
client: &'a Arc<dyn RoutedDecisionClient>,
}

struct BuiltPrivacy {
policy: PrivacyPolicy,
restricted: ExecutionLane,
}

impl<'de> Deserialize<'de> for RouteConfig {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
Expand All @@ -108,6 +123,7 @@ impl<'de> Deserialize<'de> for RouteConfig {
let tool_calling = take_optional(&mut table, "tool_calling")?;
let reasoning = take_optional(&mut table, "reasoning")?;
let vision = take_optional(&mut table, "vision")?;
let privacy = take_optional(&mut table, "privacy")?;
let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table))
.map_err(serde::de::Error::custom)?;
Ok(Self {
Expand All @@ -116,6 +132,7 @@ impl<'de> Deserialize<'de> for RouteConfig {
tool_calling,
reasoning,
vision,
privacy,
algorithm,
})
}
Expand Down Expand Up @@ -258,7 +275,7 @@ impl DeploymentConfig {
let fallback_base_url = self.fallback_base_url()?;
let mut routes = Vec::with_capacity(self.routes.len());
for (route_name, config) in &self.routes {
let targets = self.resolve_lane_targets(config)?;
let standard_targets = self.resolve_lane_targets(route_name, config, None)?;
let decision = config
.algorithm
.decision_judge()
Expand All @@ -270,9 +287,13 @@ impl DeploymentConfig {
"route {route_name} context_window must be greater than zero"
)));
}
let (lane, caller_auth) =
self.build_lane(route_name, config, &targets, decision, &clients)?;
let route = Route::from_lane(lane, caller_auth, capabilities);
let privacy = self.build_privacy(route_name, config, &standard_targets, &clients)?;
let (standard, caller_auth) =
self.build_lane(route_name, config, &standard_targets, decision, &clients)?;
let mut route = Route::from_lane(standard, caller_auth, capabilities);
if let Some(privacy) = privacy {
route = route.with_privacy(privacy.policy, privacy.restricted);
}
routes.push((config.id.clone(), route));
}
let runner = Runner::new(routes)
Expand All @@ -281,18 +302,100 @@ impl DeploymentConfig {
Ok(runner)
}

fn resolve_lane_targets<'a>(&'a self, route: &'a RouteConfig) -> RunnerResult<LaneTargets<'a>> {
route
.llm_callable_target_names()
fn build_privacy(
&self,
route_name: &str,
route: &RouteConfig,
standard: &LaneTargets<'_>,
clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
) -> RunnerResult<Option<BuiltPrivacy>> {
let Some(config) = &route.privacy else {
return Ok(None);
};
if route.algorithm.decision_judge().is_some() {
return Err(RunnerError::configuration(format!(
"route {route_name} cannot combine privacy with a typed decision judge"
)));
}
if !route.algorithm.supports_privacy_lanes() {
return Err(RunnerError::configuration(format!(
"route {route_name} cannot use privacy with prefill_router"
)));
}
if !config.accept_external_signal {
return Err(RunnerError::configuration(format!(
"route {route_name} privacy must configure at least one request input"
)));
}
let restricted_targets =
self.resolve_lane_targets(route_name, route, Some(&config.restricted_targets))?;
if self.uses_forward_auth(standard) || self.uses_forward_auth(&restricted_targets) {
return Err(RunnerError::configuration(format!(
"route {route_name} cannot use privacy with forward_auth"
)));
}
let (restricted, _) =
self.build_lane(route_name, route, &restricted_targets, None, clients)?;
Ok(Some(BuiltPrivacy {
policy: PrivacyPolicy::new(config.accept_external_signal),
restricted,
}))
}

fn uses_forward_auth(&self, targets: &LaneTargets<'_>) -> bool {
targets.values().any(|target| {
self.llm_clients
.get(&target.config.llm_client)
.is_some_and(|client| client.forward_auth)
})
}

fn resolve_lane_targets<'a>(
&'a self,
route_name: &str,
route: &'a RouteConfig,
aliases: Option<&'a BTreeMap<String, String>>,
) -> RunnerResult<LaneTargets<'a>> {
let callable = route.llm_callable_target_names();
if let Some(aliases) = aliases {
let callable_set = callable.iter().copied().collect::<BTreeSet<_>>();
if let Some(name) = callable_set
.iter()
.find(|name| !aliases.contains_key(**name))
{
return Err(RunnerError::configuration(format!(
"route {route_name} privacy is missing restricted target {name}"
)));
}
if let Some(name) = aliases
.keys()
.find(|name| !callable_set.contains(name.as_str()))
{
return Err(RunnerError::configuration(format!(
"route {route_name} privacy maps unused target {name}"
)));
}
}
callable
.into_iter()
.map(|name| {
let config = self.targets.get(name).ok_or_else(|| {
RunnerError::configuration(format!("route references unknown target {name}"))
let target_name = aliases
.and_then(|aliases| aliases.get(name).map(String::as_str))
.unwrap_or(name);
let config = self.targets.get(target_name).ok_or_else(|| {
let message = if aliases.is_some() {
format!(
"route {route_name} privacy references unknown target {target_name}"
)
} else {
format!("route references unknown target {target_name}")
};
RunnerError::configuration(message)
})?;
Ok((
name,
ResolvedTarget {
target_name: name,
target_name,
config,
},
))
Expand Down
2 changes: 2 additions & 0 deletions crates/switchyard-runner/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
mod algorithm;
mod config;
mod failure;
mod privacy;
mod provider_key_redactor;
mod route;
mod runner;
Expand All @@ -18,6 +19,7 @@ pub use algorithm::{
pub use failure::{RouteErrorKind, RouteErrorPhase, RouteErrorSummary, stream_error_summary};
// Re-exported because `Route::new` takes it, so a host wiring routes does not need a libsy dep.
pub use libsy::RuntimeModels;
pub use privacy::mark_privacy_restricted;
pub use provider_key_redactor::ProviderKeyRedactor;
pub use route::{
AuxiliaryTarget, CallerAuthKind, ModelCapabilities, Route, RunOutput, RunnerError,
Expand Down
Loading
Loading