bug(mxc): host supervisors of all MXC sandboxes share one proxy CA directory, \etc\openshell-tls #5058

# Where does the MXC host supervisor keep its proxy CA files?
#
# Starts a disposable gateway (mTLS with throwaway certificates) with an explicit MXC
# state_dir and creates one or more ProcessContainer sandboxes at the same time. Each
# workload records the proxy CA it was given and makes one HTTPS request through the
# proxy. The script then reports, per sandbox, where the supervisor's CA files are,
# whether the workload's CA matches its own sandbox's files, how many distinct CAs the
# workloads received, and what is left after `sandbox delete`. Windows PowerShell 5.1.
#
#   .\proxy-tls-dir.ps1 -BinaryDir <dir with openshell*.exe and libz3.dll> -WxcExecPath <wxc-exec.exe> [-Sandboxes 4]
param(
    [Parameter(Mandatory = $true)] [string] $BinaryDir,
    [Parameter(Mandatory = $true)] [string] $WxcExecPath,
    [int] $Sandboxes = 1,
    [int] $Port = 17760
)
$ErrorActionPreference = 'Stop'
$root = Join-Path ([IO.Path]::GetTempPath()) "openshell-tlsdir-$PID"
$work = Join-Path $root 'work'
$state = Join-Path $root 'mxc'
New-Item -ItemType Directory -Force $work | Out-Null
foreach ($name in 'APPDATA', 'LOCALAPPDATA', 'XDG_CONFIG_HOME', 'XDG_STATE_HOME', 'XDG_DATA_HOME') {
    Set-Item "Env:$name" (Join-Path $root "cli\$name")
}
$gateway = Join-Path $BinaryDir 'openshell-gateway.exe'
$cli = Join-Path $BinaryDir 'openshell.exe'
$curl = 'C:\Windows\System32\curl.exe'
$shared = Join-Path $env:SystemDrive 'etc\openshell-tls'

function Show-Shared([string] $when) {
    if (Test-Path $shared) {
        Get-ChildItem $shared | ForEach-Object {
            Write-Host "  $when $($_.FullName) $($_.LastWriteTime.ToString('HH:mm:ss.fff'))"
        }
    } else {
        Write-Host "  $when $shared does not exist"
    }
}
function Get-Pem([string] $path) {
    if (Test-Path -LiteralPath $path -PathType Leaf) { (Get-Content -LiteralPath $path -Raw).Trim() } else { $null }
}
function Quote-Arguments([string[]] $arguments) {
    ($arguments | ForEach-Object {
            if ($_ -notmatch '[\s"]') { $_ } else { '"' + (($_ -replace '(\\*)"', '$1$1\"') -replace '(\\+)$', '$1$1') + '"' }
        }) -join ' '
}
# Windows PowerShell 5.1 does not quote native arguments that contain quotes, so build the command line here.
function Start-Cli([string[]] $arguments) {
    $info = New-Object Diagnostics.ProcessStartInfo $cli, (Quote-Arguments $arguments)
    $info.UseShellExecute = $false
    $info.RedirectStandardOutput = $true
    $info.RedirectStandardError = $true
    $process = [Diagnostics.Process]::Start($info)
    [pscustomobject]@{ Process = $process; Out = $process.StandardOutput.ReadToEndAsync(); Err = $process.StandardError.ReadToEndAsync() }
}
function Invoke-Cli([string[]] $arguments) {
    $p = Start-Cli $arguments
    $p.Process.WaitForExit()
    [pscustomobject]@{ ExitCode = $p.Process.ExitCode; Output = $p.Out.Result + $p.Err.Result }
}

Write-Host '--- before'
Show-Shared 'shared'
# Throwaway CA, server and CLI client certificates, and sandbox JWT signing keys.
$certs = Join-Path $root 'certs'
& $gateway generate-certs --output-dir $certs --server-san 127.0.0.1 --server-san localhost | Out-Null
$jwt = Join-Path $certs 'jwt'
@"
[openshell]
version = 2

[openshell.gateway]
compute_driver = "mxc"
guest_tls_ca = '$certs\ca.crt'

[openshell.gateway.tls]
cert_path = '$certs\server\tls.crt'
key_path = '$certs\server\tls.key'
client_ca_path = '$certs\ca.crt'

[openshell.gateway.mtls_auth]
enabled = true

[openshell.gateway.gateway_jwt]
signing_key_path = '$jwt\signing.pem'
public_key_path = '$jwt\public.pem'
kid_path = '$jwt\kid'
gateway_id = 'tlsdir-repro'

[openshell.drivers.mxc]
allow_driver_config = true
wxc_exec_path = '$WxcExecPath'
state_dir = '$state'
"@ | Set-Content (Join-Path $root 'gateway.toml') -Encoding ASCII
@"
version: 1
filesystem_policy:
  include_workdir: false
  read_write: ['$($work.Replace('\', '/'))']
network_policies:
  example:
    name: example
    endpoints: [{ host: example.com, port: 443 }]
    binaries: [{ path: '$curl' }]
"@ | Set-Content (Join-Path $root 'policy.yaml') -Encoding ASCII

$env:OPENSHELL_GATEWAY_CONFIG = Join-Path $root 'gateway.toml'
$gw = Start-Process $gateway -PassThru -WindowStyle Hidden `
    -ArgumentList '--db-url', 'sqlite::memory:', '--port', $Port `
    -RedirectStandardOutput (Join-Path $root 'gateway.out') -RedirectStandardError (Join-Path $root 'gateway.err')
try {
    $deadline = (Get-Date).AddSeconds(30)
    while ($true) {
        try { (New-Object Net.Sockets.TcpClient('127.0.0.1', $Port)).Close(); break } catch {}
        if ((Get-Date) -gt $deadline -or $gw.HasExited) { throw 'gateway did not start' }
        Start-Sleep -Milliseconds 500
    }
    # The CLI imports the client certificate from OPENSHELL_LOCAL_TLS_DIR.
    $env:OPENSHELL_LOCAL_TLS_DIR = $certs
    $add = Invoke-Cli @('gateway', 'add', "https://127.0.0.1:$Port", '--local', '--name', 'tlsdir-repro')
    Remove-Item Env:OPENSHELL_LOCAL_TLS_DIR
    if ($add.ExitCode -ne 0) { throw "gateway add failed: $($add.Output)" }
    # OCSF JSON lines carry the supervisor's hostname; the driver forwards them to the gateway log.
    $null = Invoke-Cli @('settings', 'set', '--global', '--yes', '--key', 'ocsf_json_enabled', '--value', 'true')

    # Each workload saves the proxy CA it was given, then makes one HTTPS request through the
    # proxy that trusts only the injected bundle (curl.exe uses Schannel, which ignores
    # CURL_CA_BUNDLE without --cacert; the sandbox CA has no revocation endpoint, so revocation
    # checks are best-effort), retrying a refused CONNECT.
    $names = 1..$Sandboxes | ForEach-Object { "tlsdir-$($PID % 100000)-$_" }
    $creates = foreach ($name in $names) {
        $batch = Join-Path $work "$name.cmd"
        Set-Content $batch -Encoding ASCII -Value @(
            '@echo off',
            "copy /y `"%NODE_EXTRA_CA_CERTS%`" `"$work\$name-ca.pem`" >NUL",
            "echo SSL_CERT_FILE=%SSL_CERT_FILE%>`"$work\$name-env.txt`"",
            'set /a TRIES=0',
            ':retry',
            'set /a TRIES+=1',
            "`"$curl`" -sS --cacert `"%CURL_CA_BUNDLE%`" --ssl-revoke-best-effort -o NUL -w `"%%{http_code}`" https://example.com/ >`"$work\$name-curl.txt`" 2>&1",
            'set RC=%ERRORLEVEL%',
            'if "%RC%"=="56" if %TRIES% LSS 5 goto retry',
            'if "%RC%"=="7" if %TRIES% LSS 5 goto retry',
            "(echo  exit=%RC% tries=%TRIES%)>>`"$work\$name-curl.txt`"",
            "echo done>`"$work\$name-done.txt`"")
        $config = @{ mxc = @{ command = @('C:\Windows\System32\cmd.exe', '/d', '/c', $batch); cwd = $work } } |
            ConvertTo-Json -Compress -Depth 4
        Start-Cli @('sandbox', 'create', '--name', $name, '--no-tty',
            '--policy', (Join-Path $root 'policy.yaml'), '--driver-config-json', $config)
    }
    foreach ($c in $creates) {
        $c.Process.WaitForExit()
        if ($c.Process.ExitCode -ne 0) { Write-Host "sandbox create exit $($c.Process.ExitCode): $($c.Out.Result)$($c.Err.Result)" }
    }
    $deadline = (Get-Date).AddSeconds(90)
    while ((@($names | Where-Object { -not (Test-Path (Join-Path $work "$_-done.txt")) }).Count -gt 0) -and (Get-Date) -lt $deadline) {
        Start-Sleep -Milliseconds 500
    }

    Write-Host '--- after the workloads ran'
    Show-Shared 'shared'
    $workloadCas = @()
    foreach ($name in $names) {
        $id = ((Invoke-Cli @('sandbox', 'get', $name, '-o', 'json')).Output | ConvertFrom-Json).id
        $own = Get-ChildItem $state -Recurse -Filter 'openshell-ca.pem' -ErrorAction SilentlyContinue |
            Where-Object { $_.FullName -like "*\$id\*" } | Select-Object -First 1
        $given = Get-Pem (Join-Path $work "$name-ca.pem")
        $workloadCas += $given
        Write-Host "$name (id $id)"
        Write-Host "  CA files in its host state: $(if ($own) { $own.DirectoryName } else { 'none' })"
        if (-not $given) { Write-Host '  workload received no proxy CA' }
        if ($own -and $given) { Write-Host "  workload CA matches its own host state file: $($given -eq (Get-Pem $own.FullName))" }
        Write-Host "  workload CA matches $shared\openshell-ca.pem: $($given -and $given -eq (Get-Pem (Join-Path $shared 'openshell-ca.pem')))"
        Write-Host "  workload $(Get-Content (Join-Path $work "$name-env.txt") -ErrorAction SilentlyContinue)"
        Write-Host "  https://example.com through the proxy: $((Get-Content (Join-Path $work "$name-curl.txt") -Raw -ErrorAction SilentlyContinue) -replace '\s+', ' ')"
    }
    Write-Host "distinct proxy CAs across $Sandboxes workloads: $(@($workloadCas | Where-Object { $_ } | Sort-Object -Unique).Count)"
    $failed = @(Get-Content (Join-Path $root 'gateway.out'), (Join-Path $root 'gateway.err') -ErrorAction SilentlyContinue |
            Where-Object { $_ -match 'Failed to write CA files' -and $_ -notmatch 'OCSF-JSON' })
    Write-Host "supervisors that logged 'Failed to write CA files, TLS termination disabled': $($failed.Count)"
    $failed | Select-Object -First 3 | ForEach-Object { Write-Host "  $(($_ -replace '\x1b\[[0-9;]*m', '') -replace '^.*?(Failed to write CA files)', '$1')" }
    # The driver forwards the host supervisor's stderr, including its OCSF-JSON lines, to the gateway log.
    $hostnames = Get-Content (Join-Path $root 'gateway.out'), (Join-Path $root 'gateway.err') -ErrorAction SilentlyContinue |
        Where-Object { $_ -match 'OCSF-JSON' -and $_ -match 'supervisor' } |
        ForEach-Object { [regex]::Matches($_, 'device\\?":\{\\?"hostname\\?":\\?"([^"\\]+)') } |
        ForEach-Object { $_.Groups[1].Value } | Sort-Object -Unique
    Write-Host "OCSF hostname in the host supervisor's events: $($hostnames -join ', ') (DNS host name $([Net.Dns]::GetHostName()), COMPUTERNAME $env:COMPUTERNAME)"

    foreach ($name in $names) { $null = Invoke-Cli @('sandbox', 'delete', $name) }
    Start-Sleep -Seconds 2
    Write-Host '--- after sandbox delete'
    Show-Shared 'shared'
    Write-Host "  CA files left under ${state}: $(@(Get-ChildItem $state -Recurse -Filter '*.pem' -ErrorAction SilentlyContinue | Where-Object { $_.Name -in 'openshell-ca.pem', 'ca-bundle.pem' }).Count)"
    Write-Host "repro state: $root"
} finally {
    if (-not $gw.HasExited) { Stop-Process -Id $gw.Id -Force }
}