Feature/tiered storage

.github/workflows/unit-test.yml

@@ -201,6 +201,81 @@ jobs:
             go test -v -timeout=1h ./... --tags=unittest,azurite,fuse3
           fi
 
+      - name: Tiered storage mount smoke test
+        if: matrix.job_name == 'linux-fuse3'
+        env:
+          S3_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+          S3_SECRET_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          S3_ENDPOINT: ${{ env.AWS_ENDPOINT }}
+          S3_REGION: ${{ env.AWS_REGION }}
+          S3_BUCKET_NAME: ${{ env.AWS_BUCKET_NAME }}
+        run: |
+          set -euo pipefail
+          MOUNT_DIR=/tmp/cloudfuse-tiered-mount
+          TIER_DIR=/tmp/cloudfuse-tiered-data
+          CONFIG=/tmp/cloudfuse-tiered.yaml
+          SOURCE=/tmp/cloudfuse-tiered-source
+          PERSISTENT=/tmp/cloudfuse-tiered-persistent
+          MOUNT_PID=
+
+          cleanup() {
+            if mountpoint -q "$MOUNT_DIR"; then
+              fusermount3 -u "$MOUNT_DIR" -z || true
+            fi
+            if [[ -n "$MOUNT_PID" ]]; then
+              kill "$MOUNT_PID" 2>/dev/null || true
+            fi
+            rm -rf "$MOUNT_DIR" "$TIER_DIR" "$CONFIG" "$SOURCE" "$PERSISTENT"
+          }
+          trap cleanup EXIT
+
+          mkdir -p "$MOUNT_DIR" "$TIER_DIR"
+          ./cloudfuse gen-test-config \
+            --config-file=s3_key_tiered_storage.yaml \
+            --temp-path="$TIER_DIR" \
+            --output-file="$CONFIG"
+
+          mount_tiered() {
+            ./cloudfuse mount "$MOUNT_DIR" --config-file="$CONFIG" --foreground=true &
+            MOUNT_PID=$!
+            for _ in {1..30}; do
+              mountpoint -q "$MOUNT_DIR" && return
+              sleep 1
+            done
+            return 1
+          }
+
+          unmount_tiered() {
+            fusermount3 -u "$MOUNT_DIR"
+            wait "$MOUNT_PID"
+            MOUNT_PID=
+          }
+
+          mount_tiered
+          head -c 900000 /dev/urandom > "$SOURCE"
+          cp "$SOURCE" "$MOUNT_DIR/evicted.bin"
+          for _ in {1..30}; do
+            aws --endpoint-url "$AWS_ENDPOINT" s3api head-object \
+              --bucket "$AWS_BUCKET_NAME" --key evicted.bin >/dev/null 2>&1 && break
+            sleep 1
+          done
+          aws --endpoint-url "$AWS_ENDPOINT" s3api head-object \
+            --bucket "$AWS_BUCKET_NAME" --key evicted.bin >/dev/null
+          test ! -e "$TIER_DIR/evicted.bin"
+          cmp "$SOURCE" "$MOUNT_DIR/evicted.bin"
+          test ! -e "$TIER_DIR/evicted.bin"
+
+          head -c 100000 /dev/urandom > "$PERSISTENT"
+          cp "$PERSISTENT" "$MOUNT_DIR/persistent.bin"
+          test -e "$TIER_DIR/persistent.bin"
+          unmount_tiered
+
+          mount_tiered
+          test -e "$TIER_DIR/persistent.bin"
+          cmp "$PERSISTENT" "$MOUNT_DIR/persistent.bin"
+          rm "$MOUNT_DIR/persistent.bin"
+          unmount_tiered
+
   test:
     name: Build and Test on Windows
     environment: testing

README.md

@@ -22,7 +22,7 @@
 
 Cloudfuse provides the ability to mount a cloud bucket in your local filesystem on Linux and Windows.
 With Cloudfuse you can easily read and write to the cloud, and connect programs on your computer to the cloud even if they're not cloud-aware.
-Cloudfuse uses file caching to provide the performance of local storage, or you can use streaming mode to efficiently access small parts of large files (e.g. video playback).
+Cloudfuse can run in two modes: caching, where the cloud is the source of truth and local disk holds a temporary copy for speed, or tiered storage, where local disk is the primary copy and the cloud is overflow for data that no longer fits locally.
 Cloudfuse is a fork of [blobfuse2](https://github.com/Azure/azure-storage-fuse), and adds S3 support and Windows support.
 Cloudfuse supports clouds with an S3 or Azure interface.
 
@@ -145,10 +145,10 @@ manually install Cloudfuse.
 
 ## Basic Use
 
-The following describes how to use the Cloudfuse CLI. If you would like to use a GUI checkout the Cloudfuse GUI repo at <https://github.com/Seagate/cloudfuse-gui>.
+The following describes how to use the Cloudfuse CLI.
 
 1. Create a basic configuration file (TUI):
-   If you would like an easy way to get started with cloudfuse, run the following to launch a TUI to configure cloudfuse. If you prefer to configure manually, checkout how to write a config file: <https://github.com/Seagate/cloudfuse/wiki/Config-File>
+   If you would like an easy way to get started with cloudfuse, run the following to launch a TUI to configure cloudfuse. The TUI does not currently support tiering. If you prefer to configure manually, checkout how to write a config file: <https://github.com/Seagate/cloudfuse/wiki/Config-File>
 
    ```bash
    cloudfuse config
@@ -314,6 +314,21 @@ Cloudfuse now supports offline access through the `file_cache` component. When c
 
 > **Note:** Cloudfuse uses eventual consistency with last-writer-wins semantics. Offline access can extend the consistency window indefinitely and **increases the risk of data conflicts in multi-client setups!** See [component/file_cache/OfflineAccess.md](component/file_cache/OfflineAccess.md) for full details and configuration guidance.
 
+## Tiered Storage
+
+Use `tiered_storage` for on-prem-first configurations, when you want cloud storage to expand existing local storage. To minimize cloud storage costs, `tiered_storage` only maintains one copy of each file, moving old files to cloud storage once local storage fills up.
+
+Note: Do not use `tiered_storage` if you use cloud storage to access your data remotely. Only overflow data will be present in the cloud.
+
+See `sample_configs/sampleTieredStorageConfigS3.yaml` and `setup/baseConfig.yaml` for configuration.
+
+### Expanding Existing Local Storage Using Tiered Storage
+
+When using `tiered_storage` to add cloud capacity to an existing local storage location, please ensure all access is done through the Cloudfuse mount path / virtual drive. The `tiered_storage` `path` and all its existing contents will become *internal* local storage. Accessing the `tiered_storage` `path` directly may have unpredictable results.
+For example, if you want expand the capacity of your existing local drive, `D:`, you could change that local drive's letter to `E:` first, then set `tiered_storage` `path` to `E:` and mount cloudfuse to `D:`. Then any existing applications or workflows that normally use `D:` would continue as normal, now pointed at the cloudfuse mount location.
+
+**Cloudfuse may upload and delete local files under `path` as part of normal overflow eviction, so anything still reading or writing there directly can lose data or see missing files.**
+
 ## Limitations
 
 ### NOTICE

cmd/imports.go

@@ -29,11 +29,14 @@ import (
 	_ "github.com/Seagate/cloudfuse/component/attr_cache"
 	_ "github.com/Seagate/cloudfuse/component/azstorage"
 	_ "github.com/Seagate/cloudfuse/component/block_cache"
+	_ "github.com/Seagate/cloudfuse/component/custom"
+	_ "github.com/Seagate/cloudfuse/component/entry_cache"
 	_ "github.com/Seagate/cloudfuse/component/file_cache"
 	_ "github.com/Seagate/cloudfuse/component/libfuse"
 	_ "github.com/Seagate/cloudfuse/component/loopback"
 	_ "github.com/Seagate/cloudfuse/component/s3storage"
 	_ "github.com/Seagate/cloudfuse/component/size_tracker"
 	_ "github.com/Seagate/cloudfuse/component/stream"
+	_ "github.com/Seagate/cloudfuse/component/tiered_storage"
 	_ "github.com/Seagate/cloudfuse/component/xload"
 )

cmd/mount_linux_test.go

@@ -804,6 +804,17 @@ func (suite *mountTestSuite) TestCleanUpOnStartFlag() {
 		suite.assert.False(common.IsDirectoryEmpty(cachedirs[(i+1)%3]))
 		suite.assert.False(common.IsDirectoryEmpty(cachedirs[(i+2)%3]))
 	}
+
+	tieredPath := filepath.Join(testDir, "tiered")
+	err = os.MkdirAll(tieredPath, 0755)
+	suite.assert.NoError(err)
+	err = os.WriteFile(filepath.Join(tieredPath, "authoritative"), []byte("data"), 0644)
+	suite.assert.NoError(err)
+	config.Set("tiered_storage.path", tieredPath)
+	config.Set("cleanup-on-start", "true")
+	options.Components = []string{"tiered_storage"}
+	suite.assert.NoError(options.tempCacheCleanup())
+	suite.assert.False(common.IsDirectoryEmpty(tieredPath))
 }
 
 // TestValidateMountOptionsInvalidLogLevel tests validation with invalid log level

common/lock_map.go

@@ -33,7 +33,7 @@ import (
 
 // Lock item for each file
 type LockMapItem struct {
-	handleCount  uint32
+	handleCount  atomic.Uint32
 	dirtyCount   atomic.Uint32
 	mtx          sync.RWMutex
 	downloadTime time.Time
@@ -54,7 +54,7 @@ func NewLockMap() *LockMap {
 
 // Get the lock item based on file name, if item does not exists create it
 func (l *LockMap) Get(name string) *LockMapItem {
-	lockIntf, _ := l.locks.LoadOrStore(name, &LockMapItem{handleCount: 0})
+	lockIntf, _ := l.locks.LoadOrStore(name, &LockMapItem{})
 	item := lockIntf.(*LockMapItem)
 	return item
 }
@@ -75,6 +75,10 @@ func (l *LockMapItem) Unlock() {
 	l.mtx.Unlock()
 }
 
+func (l *LockMapItem) TryLock() bool {
+	return l.mtx.TryLock()
+}
+
 func (l *LockMapItem) RLock() {
 	l.mtx.RLock()
 }
@@ -86,17 +90,25 @@ func (l *LockMapItem) RUnlock() {
 
 // Increment the handle count
 func (l *LockMapItem) Inc() {
-	l.handleCount++
+	l.handleCount.Add(1)
 }
 
 // Decrement the handle count
 func (l *LockMapItem) Dec() {
-	l.handleCount--
+	for {
+		current := l.handleCount.Load()
+		if current == 0 {
+			return
+		}
+		if l.handleCount.CompareAndSwap(current, current-1) {
+			return
+		}
+	}
 }
 
 // Get the current handle count
 func (l *LockMapItem) Count() uint32 {
-	return l.handleCount
+	return l.handleCount.Load()
 }
 
 // Increment dirty-handle count.

common/util.go

@@ -618,20 +618,14 @@ func ComponentInPipeline(pipeline []string, component string) bool {
 }
 
 func ValidatePipeline(pipeline []string) error {
-	// file-cache, block-cache and xload are mutually exclusive
-	if ComponentInPipeline(pipeline, "file_cache") &&
-		ComponentInPipeline(pipeline, "block_cache") {
-		return fmt.Errorf("mount: file-cache and block-cache cannot be used together")
-	}
-
-	if ComponentInPipeline(pipeline, "file_cache") &&
-		ComponentInPipeline(pipeline, "xload") {
-		return fmt.Errorf("mount: file-cache and xload cannot be used together")
+	var selected []string
+	for _, component := range []string{"file_cache", "block_cache", "xload", "tiered_storage"} {
+		if ComponentInPipeline(pipeline, component) {
+			selected = append(selected, component)
+		}
 	}
-
-	if ComponentInPipeline(pipeline, "block_cache") &&
-		ComponentInPipeline(pipeline, "xload") {
-		return fmt.Errorf("mount: block-cache and xload cannot be used together")
+	if len(selected) > 1 {
+		return fmt.Errorf("mount: %s cannot be used together", strings.Join(selected, ", "))
 	}
 
 	return nil

common/util_test.go

@@ -665,6 +665,15 @@ func (suite *utilTestSuite) TestValidatePipeline() {
 	err = ValidatePipeline([]string{"libfuse", "file_cache", "block_cache", "xload", "azstorage"})
 	suite.Error(err)
 
+	err = ValidatePipeline([]string{"libfuse", "file_cache", "tiered_storage", "azstorage"})
+	suite.Error(err)
+
+	err = ValidatePipeline([]string{"libfuse", "block_cache", "tiered_storage", "azstorage"})
+	suite.Error(err)
+
+	err = ValidatePipeline([]string{"libfuse", "xload", "tiered_storage", "azstorage"})
+	suite.Error(err)
+
 	err = ValidatePipeline([]string{"libfuse", "file_cache", "azstorage"})
 	suite.NoError(err)
 
@@ -673,6 +682,9 @@ func (suite *utilTestSuite) TestValidatePipeline() {
 
 	err = ValidatePipeline([]string{"libfuse", "xload", "attr_cache", "azstorage"})
 	suite.NoError(err)
+
+	err = ValidatePipeline([]string{"libfuse", "tiered_storage", "attr_cache", "azstorage"})
+	suite.NoError(err)
 }
 
 func (suite *utilTestSuite) TestUpdatePipeline() {

component/tiered_storage/cache_size.go

@@ -0,0 +1,114 @@
+/*
+   Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+
+   Copyright © 2026 Seagate Technology LLC and/or its Affiliates
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in all
+   copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE
+*/
+
+package tiered_storage
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/Seagate/cloudfuse/common"
+	"github.com/Seagate/cloudfuse/common/log"
+)
+
+// driftWarnBytes is how far the running total may diverge from a measurement
+// before the correction is worth a log line.
+const driftWarnBytes = 16 * common.MbToBytes
+
+// cacheSizeTracker tracks how many bytes of local storage the component is
+// using.
+//
+// The total is maintained incrementally, because measuring it costs a du(1)
+// subprocess on Linux and a full tree walk on Windows - far too expensive to do
+// per write or per eviction tick. Incremental accounting drifts (directory
+// entries, sparse files, anything that touches the cache directory behind our
+// back), so Reconcile periodically replaces the total with a real measurement.
+//
+// Sizes are apparent bytes, matching `du -sb` on Linux. Windows measures
+// allocated sectors instead, so the two platforms disagree slightly for small
+// files; Reconcile is what keeps that from accumulating.
+type cacheSizeTracker struct {
+	path              string
+	used              atomic.Int64
+	reconcileInterval time.Duration
+	lastReconcile     atomic.Int64 // unix nanoseconds
+}
+
+func newCacheSizeTracker(path string, reconcileInterval time.Duration) *cacheSizeTracker {
+	t := &cacheSizeTracker{path: path, reconcileInterval: reconcileInterval}
+	t.lastReconcile.Store(time.Now().UnixNano())
+	return t
+}
+
+// Used returns the current cache usage in bytes.
+func (t *cacheSizeTracker) Used() int64 {
+	return t.used.Load()
+}
+
+// Add applies a signed change in bytes. The total is clamped at zero: drift
+// could otherwise drive it negative and hide a full cache.
+func (t *cacheSizeTracker) Add(delta int64) {
+	if delta == 0 {
+		return
+	}
+	for {
+		current := t.used.Load()
+		next := max(current+delta, 0)
+		if t.used.CompareAndSwap(current, next) {
+			return
+		}
+	}
+}
+
+// Refresh measures the cache directory and replaces the running total.
+// Changes made while the measurement is in flight are lost, which is inherent
+// to reconciling against a moving target and is why it is not the hot path.
+func (t *cacheSizeTracker) Refresh() {
+	t.lastReconcile.Store(time.Now().UnixNano())
+
+	usage, err := common.GetUsage(t.path)
+	if err != nil {
+		log.Err("cacheSizeTracker::Refresh : failed to measure %s [%v]", t.path, err)
+		return
+	}
+
+	measured := int64(usage)
+	previous := t.used.Swap(measured)
+	if drift := measured - previous; drift > driftWarnBytes || drift < -driftWarnBytes {
+		log.Info(
+			"cacheSizeTracker::Refresh : corrected usage by %d bytes (was %d, now %d)",
+			drift,
+			previous,
+			measured,
+		)
+	}
+}
+
+// Reconcile refreshes the running total, but no more often than reconcileInterval.
+func (t *cacheSizeTracker) Reconcile() {
+	if time.Since(time.Unix(0, t.lastReconcile.Load())) < t.reconcileInterval {
+		return
+	}
+	t.Refresh()
+}