Skip to content

remove bbolt integrity check entirely #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Trent-TSE wants to merge 2 commits into
base: main
Choose a base branch
from
Open

remove bbolt integrity check entirely #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
422c214
fix: treat RuntimeMaxSec SIGTERM (exit 143) in bbolt check as timeout…
Trent-TSE Jul 15, 2026
36a638d
remove bbolt integrity check entirely
Trent-TSE Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ This action follows semantic versioning and supports multiple referencing patter
| `platforms` | List of target platforms for build (e.g., linux/amd64,linux/arm64) | No | |
| `nofallback` | If true, fail the action if Blacksmith builder setup fails | No | `false` |
| `github-token` | GitHub token for GitHub API access | No | |
| `skip-integrity-check` | If true, skip the bbolt database integrity check | No | `false` |
| `skip-integrity-check` | Deprecated: the bbolt database integrity check has been removed; this input has no effect | No | `false` |
| `driver-opts` | List of additional driver-specific options (e.g., env.VARIABLE=value) | No | |
| `max-parallelism` | Maximum number of concurrent BuildKit RUN steps. Defaults to the number of vCPUs on the runner | No | |
| `max-cache-size-mb` | Amount of build cache to retain after pruning, in MB (e.g., 409600 for 400GB). If not set, pruning is skipped | No | |
Expand Down
2 changes: 1 addition & 1 deletion action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ inputs:
description: "BuildKit version to install (e.g., v0.16.0, v0.18.0). If not specified, uses system default"
required: false
skip-integrity-check:
description: "If true, skip the bbolt database integrity check"
description: "Deprecated: the bbolt database integrity check has been removed; this input has no effect"
required: false
default: "false"
driver-opts:
Expand Down
2 changes: 1 addition & 1 deletion dist/index.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dist/index.js.map
View file
Open in desktop

Large diffs are not rendered by default.

192 changes: 2 additions & 190 deletions src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,6 @@ import { exec } from "child_process";

import * as stateHelper from "./state-helper";
import * as reporter from "./reporter";
import {
execWithTimeout,
ExecTimeoutError,
BOLT_CHECK_MEMORY_MAX_BYTES,
BOLT_CHECK_MAX_FILE_BYTES,
} from "./exec-utils";
import {
setupStickyDisk,
startAndConfigureBuildkitd,
Expand Down Expand Up @@ -147,173 +141,13 @@ async function flushBlockDevice(devicePath: string): Promise<void> {
}
}

async function checkBoltDbIntegrity(skip = false): Promise<boolean> {
if (skip) {
core.info(
"Skipping bbolt database integrity check (skip-integrity-check is enabled)",
);
return true;
}

try {
// Check if /var/lib/buildkit directory exists
try {
await execWithTimeout(
"test -d /var/lib/buildkit",
15_000,
"test buildkit dir exists",
);
core.debug(
"Found /var/lib/buildkit directory, checking for database files",
);

// Find all *.db files in /var/lib/buildkit
const { stdout: dbFiles } = await execWithTimeout(
"find /var/lib/buildkit -name '*.db' 2>/dev/null || true",
30_000,
"find db files",
);

if (dbFiles.trim()) {
const files = dbFiles.trim().split("\n");
core.info(
`Found ${files.length} database file(s): ${files.join(", ")}`,
);

let allChecksPass = true;
for (const dbFile of files) {
if (dbFile.trim()) {
try {
// Get file size
let sizeInfo = "";
let sizeBytes = 0;
try {
const { stdout: sizeOutput } = await execWithTimeout(
`stat -c%s "${dbFile}" 2>/dev/null || stat -f%z "${dbFile}"`,
15_000,
`stat db file ${dbFile}`,
);
sizeBytes = parseInt(sizeOutput.trim(), 10);
if (!isNaN(sizeBytes) && sizeBytes > 0) {
const sizeMB = (sizeBytes / (1024 * 1024)).toFixed(2);
sizeInfo = ` (${sizeMB} MB)`;
}
} catch (error) {
core.debug(
`Could not determine file size for ${dbFile}: ${(error as Error).message}`,
);
}

// Skip integrity check for files that are too large for the memory-limited
// systemd scope. bbolt check mmaps the entire file, and with ~50-60 MB of
// Go runtime overhead the process will be OOM-killed for large files.
if (sizeBytes > BOLT_CHECK_MAX_FILE_BYTES) {
const sizeMB = (sizeBytes / (1024 * 1024)).toFixed(2);
core.info(
`${dbFile}: Skipping integrity check - file size ${sizeMB} MB exceeds limit (${BOLT_CHECK_MAX_FILE_BYTES / (1024 * 1024)} MB)`,
);
continue;
}

core.info(`Running bolt check on ${dbFile}${sizeInfo}...`);
const startTime = Date.now();

try {
const memoryMaxMB = BOLT_CHECK_MEMORY_MAX_BYTES / (1024 * 1024);
const { stdout: checkResult } = await execWithTimeout(
`sudo systemd-run --scope --quiet -p MemoryMax=${memoryMaxMB}M -p RuntimeMaxSec=6s bbolt check "${dbFile}" 2>&1`,
30_000,
`bbolt check ${dbFile}`,
);
const duration = Date.now() - startTime;
const durationSeconds = (duration / 1000).toFixed(2);

if (duration > 5000) {
core.warning(
`⚠ ${dbFile}: Check took ${durationSeconds}s (exceeded 5s threshold)`,
);
}

if (checkResult.includes("OK")) {
core.info(`✓ ${dbFile}: Database integrity check passed`);
} else {
core.warning(`⚠ ${dbFile}: ${checkResult}`);
allChecksPass = false;
// Report failed check
await reporter.reportIntegrityCheckFailure(dbFile);
}
} catch (checkError) {
const duration = Date.now() - startTime;
const durationSeconds = (duration / 1000).toFixed(2);
const exitCode = (checkError as { code?: number }).code;
const errorMessage = (checkError as Error).message;

// ExecTimeoutError = Promise.race timeout (process stuck in D state, e.g. Ceph partition)
if (checkError instanceof ExecTimeoutError) {
core.warning(
`⚠ ${dbFile}: Integrity check hit hard timeout after ${durationSeconds}s (possible I/O stall) - skipping`,
);
// Exit code 124 = timeout, 137 = SIGKILL (likely OOM), 143 = SIGTERM
} else if (exitCode === 124) {
core.warning(
`⚠ ${dbFile}: Integrity check timed out after ${durationSeconds}s - skipping`,
);
} else if (
exitCode === 137 ||
errorMessage.toLowerCase().includes("out of memory") ||
errorMessage.toLowerCase().includes("cannot allocate memory")
) {
core.warning(
`⚠ ${dbFile}: Integrity check hit memory limit - skipping`,
);
} else {
core.warning(
`⚠ ${dbFile}: Integrity check failed: ${errorMessage}`,
);
allChecksPass = false;
// Report actual failure
await reporter.reportIntegrityCheckFailure(dbFile);
}
}
} catch (error) {
core.warning(
`Failed to check ${dbFile}: ${(error as Error).message}`,
);
allChecksPass = false;
}
}
}
return allChecksPass;
} else {
core.info("No *.db files found in /var/lib/buildkit");
return true;
}
} catch (error) {
if (error instanceof ExecTimeoutError) {
core.warning(
`Integrity check hit hard timeout during filesystem access (possible I/O stall) - skipping`,
);
return true;
}
core.info(
`/var/lib/buildkit directory not found, skipping database checks ${(error as Error).message}`,
);
return true;
}
} catch (error) {
core.warning(`BoltDB check failed: ${(error as Error).message}`);
return false;
}
}

// Minimal inputs interface for setup-docker-builder
export interface Inputs {
"buildx-version": string;
"buildkit-version": string;
platforms: string[];
nofallback: boolean;
"github-token": string;
"skip-integrity-check": boolean;
"driver-opts": string[];
"max-parallelism": number | null;
}
Expand All @@ -338,7 +172,6 @@ async function getInputs(): Promise<Inputs> {
platforms: Util.getInputList("platforms"),
nofallback: core.getBooleanInput("nofallback"),
"github-token": core.getInput("github-token"),
"skip-integrity-check": core.getBooleanInput("skip-integrity-check"),
"driver-opts": Util.getInputList("driver-opts", {
ignoreComma: true,
quote: false,
Expand Down Expand Up @@ -485,14 +318,6 @@ async function startBlacksmithBuilder(
parallelism = inputs["max-parallelism"];
}

// Check for potential boltdb corruption
const boltdbIntegrity = await checkBoltDbIntegrity(
inputs["skip-integrity-check"],
);
if (!boltdbIntegrity) {
core.error("BoltDB integrity check failed");
}

// Start buildkitd
const buildkitdStartTime = Date.now();
let buildkitdAddr: string;
Expand Down Expand Up @@ -768,7 +593,6 @@ void actionsToolkit.run(
const exposeId = stateHelper.getExposeId();
let cleanupError: Error | null = null;
let fsDiskUsageBytes: number | null = null;
let integrityCheckPassed: boolean | null = null;

try {
// Step 1: Shut down buildkitd if this instance started it.
Expand Down Expand Up @@ -797,12 +621,8 @@ void actionsToolkit.run(
const { stdout: mountOutput } = await execAsync(
`mount | grep "${mountPoint}"`,
);
integrityCheckPassed = await checkBoltDbIntegrity(
stateHelper.inputs?.["skip-integrity-check"] ?? false,
);

// Log database file hashes after integrity check
await logDatabaseHashes("after integrity check");
// Log database file hashes before unmount
await logDatabaseHashes("before unmount");

// Get filesystem usage BEFORE unmounting (critical timing)
try {
Expand Down Expand Up @@ -919,14 +739,6 @@ void actionsToolkit.run(
core.warning(
"Skipping sticky disk commit due to ambiguity in failure detection",
);
} else if (integrityCheckPassed === null) {
core.warning(
"Skipping sticky disk commit due to integrity check not being run",
);
} else if (!integrityCheckPassed) {
core.warning(
"Skipping sticky disk commit due to integrity check failure",
);
} else if (failureCheck.hasFailures) {
core.warning(
`Found ${failureCheck.failedCount} failed/cancelled steps in previous workflow steps`,
Expand Down
46 changes: 0 additions & 46 deletions src/reporter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,52 +158,6 @@ export async function reportMetric(
}
}

/**
* Reports bolt DB integrity check failure to the FA agent's internal metrics endpoint.
* This sends the metric to Grafana via OpenTelemetry with the database file as an attribute.
* Only called when an integrity check fails.
*/
export async function reportIntegrityCheckFailure(
dbFile: string,
): Promise<void> {
try {
const metricsPort =
process.env.BLACKSMITH_METRICS_HTTP_PORT ||
process.env.METRICS_PORT ||
"5556";
const metricsHost = "192.168.127.1";
const url = `http://${metricsHost}:${metricsPort}/internal`;

// Extract database file name (e.g., "history.db" or "cache.db")
const dbFileName = dbFile.split("/").pop() || dbFile;

const payload = {
metric_type: "boltdb_integrity_check_failure",
value: 1, // Always 1 for failures
vm_id: process.env.BLACKSMITH_VM_ID || "",
attributes: {
database_file: dbFileName,
},
};

const response = await axios.post(url, payload, {
timeout: 2000, // 2 second timeout
headers: {
"Content-Type": "application/json",
},
});

core.debug(
`Reported integrity check failure for ${dbFileName} (${response.status})`,
);
} catch (error) {
// Don't fail the action if metrics reporting fails
core.warning(
`Failed to report integrity check metric: ${(error as Error).message}`,
);
}
}

export async function commitStickyDisk(
exposeId: string,
fsDiskUsageBytes: number | null,
Expand Down
Loading