From 318a10946596056e6844621fa0d593addfa2de4b Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Wed, 27 May 2026 10:57:07 -0600 Subject: [PATCH 01/89] Add tiered storage component --- cmd/imports.go | 3 + component/tiered_storage/tiered_storage.go | 242 +++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 component/tiered_storage/tiered_storage.go diff --git a/cmd/imports.go b/cmd/imports.go index e253a2a75..b1bbb9314 100644 --- a/cmd/imports.go +++ b/cmd/imports.go @@ -29,11 +29,14 @@ import ( _ "github.com/Seagate/cloudfuse/component/attr_cache" _ "github.com/Seagate/cloudfuse/component/azstorage" _ "github.com/Seagate/cloudfuse/component/block_cache" + _ "github.com/Seagate/cloudfuse/component/custom" + _ "github.com/Seagate/cloudfuse/component/entry_cache" _ "github.com/Seagate/cloudfuse/component/file_cache" _ "github.com/Seagate/cloudfuse/component/libfuse" _ "github.com/Seagate/cloudfuse/component/loopback" _ "github.com/Seagate/cloudfuse/component/s3storage" _ "github.com/Seagate/cloudfuse/component/size_tracker" _ "github.com/Seagate/cloudfuse/component/stream" + _ "github.com/Seagate/cloudfuse/component/tiered_storage" _ "github.com/Seagate/cloudfuse/component/xload" ) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go new file mode 100644 index 000000000..913c001d5 --- /dev/null +++ b/component/tiered_storage/tiered_storage.go @@ -0,0 +1,242 @@ +/* + Licensed under the MIT License . + + Copyright © 2023-2026 Seagate Technology LLC and/or its Affiliates + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package tiered_storage + +import ( + "context" + "fmt" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/config" + "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/internal" + "github.com/Seagate/cloudfuse/internal/handlemap" +) + +/* NOTES: + - Component shall have a structure which inherits "internal.BaseComponent" to participate in pipeline + - Component shall register a name and its constructor to participate in pipeline (add by default by generator) + - Order of calls : Constructor -> Configure -> Start ..... -> Stop + - To read any new setting from config file follow the Configure method default comments +*/ + +// Common structure for Component +type TieredStorage struct { + internal.BaseComponent +} + +// Structure defining your config parameters +type TieredStorageOptions struct { + // e.g. var1 uint32 `config:"var1"` +} + +const compName = "tiered_storage" + +// Verification to check satisfaction criteria with Component Interface +var _ internal.Component = &TieredStorage{} + +func (c *TieredStorage) Name() string { + return compName +} + +func (c *TieredStorage) SetName(name string) { + c.BaseComponent.SetName(name) +} + +func (c *TieredStorage) SetNextComponent(nc internal.Component) { + c.BaseComponent.SetNextComponent(nc) +} + +// Start : Pipeline calls this method to start the component functionality +// +// this shall not block the call otherwise pipeline will not start +func (c *TieredStorage) Start(ctx context.Context) error { + log.Trace("TieredStorage::Start : Starting component %s", c.Name()) + + // TieredStorage : start code goes here + + return nil +} + +// Stop : Stop the component functionality and kill all threads started +func (c *TieredStorage) Stop() error { + log.Trace("TieredStorage::Stop : Stopping component %s", c.Name()) + + return nil +} + +// Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself +// +// Return failure if any config is not valid to exit the process +func (c *TieredStorage) Configure(_ bool) error { + log.Trace("TieredStorage::Configure : %s", c.Name()) + + // >> If you do not need any config parameters remove below code and return nil + conf := TieredStorageOptions{} + err := config.UnmarshalKey(c.Name(), &conf) + if err != nil { + log.Err("TieredStorage::Configure : config error [invalid config attributes]") + return fmt.Errorf("TieredStorage: config error [invalid config attributes]") + } + // Extract values from 'conf' and store them as you wish here + + return nil +} + +// OnConfigChange : If component has registered, on config file change this method is called +func (c *TieredStorage) OnConfigChange() { +} + +// Directory operations +func (c *TieredStorage) CreateDir(options internal.CreateDirOptions) error { + return nil +} + +func (c *TieredStorage) DeleteDir(options internal.DeleteDirOptions) error { + return nil +} + +func (c *TieredStorage) IsDirEmpty(options internal.IsDirEmptyOptions) bool { + return false +} + +func (c *TieredStorage) OpenDir(options internal.OpenDirOptions) error { + return nil +} + +func (c *TieredStorage) StreamDir( + options internal.StreamDirOptions, +) ([]*internal.ObjAttr, string, error) { + return nil, "", nil +} + +func (c *TieredStorage) CloseDir(options internal.CloseDirOptions) error { + return nil +} + +func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { + return nil +} + +// File operations +func (c *TieredStorage) CreateFile( + options internal.CreateFileOptions, +) (*handlemap.Handle, error) { + return nil, nil +} + +func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { + return nil +} + +func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error) { + return nil, nil +} + +func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int, error) { + return 0, nil +} + +func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { + return 0, nil +} + +func (c *TieredStorage) SyncFile(options internal.SyncFileOptions) error { + return nil +} + +func (c *TieredStorage) FlushFile(options internal.FlushFileOptions) error { + return nil +} + +func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { + return nil +} + +func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { + return nil +} + +func (c *TieredStorage) CopyToFile(options internal.CopyToFileOptions) error { + return nil +} + +func (c *TieredStorage) CopyFromFile(options internal.CopyFromFileOptions) error { + return nil +} + +func (c *TieredStorage) SyncDir(options internal.SyncDirOptions) error { + return nil +} + +// Symlink operations +func (c *TieredStorage) CreateLink(options internal.CreateLinkOptions) error { + return nil +} + +func (c *TieredStorage) ReadLink(options internal.ReadLinkOptions) (string, error) { + return "", nil +} + +// Filesystem level operations +func (c *TieredStorage) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { + return &internal.ObjAttr{}, nil +} + +func (c *TieredStorage) Chmod(options internal.ChmodOptions) error { + return nil +} + +func (c *TieredStorage) Chown(options internal.ChownOptions) error { + return nil +} + +func (c *TieredStorage) TruncateFile(options internal.TruncateFileOptions) error { + return nil +} + +func (c *TieredStorage) FileUsed(name string) error { + return nil +} + +func (c *TieredStorage) StatFs() (*common.Statfs_t, bool, error) { + return nil, false, nil +} + +// ------------------------- Factory ------------------------------------------- + +// Pipeline will call this method to create your object, initialize your variables here +// << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> +func NewTieredStorageComponent() internal.Component { + comp := &TieredStorage{} + comp.SetName(compName) + return comp +} + +// On init register this component to pipeline and supply your constructor +func init() { + internal.AddComponent(compName, NewTieredStorageComponent) +} From 0d5cbf31f1499d7cd5cd71e624ce9d1dc0f60e6c Mon Sep 17 00:00:00 2001 From: James Fantin-Hardesty <24646452+jfantinhardesty@users.noreply.github.com> Date: Wed, 27 May 2026 10:59:54 -0600 Subject: [PATCH 02/89] Remove unused functions --- component/tiered_storage/tiered_storage.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 913c001d5..97f6c7b4c 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -180,14 +180,6 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { return nil } -func (c *TieredStorage) CopyToFile(options internal.CopyToFileOptions) error { - return nil -} - -func (c *TieredStorage) CopyFromFile(options internal.CopyFromFileOptions) error { - return nil -} - func (c *TieredStorage) SyncDir(options internal.SyncDirOptions) error { return nil } From 7d93b2af5e2ec755ae2a80c572368a7b83572ab9 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 2 Jun 2026 15:41:38 -0600 Subject: [PATCH 03/89] Initial structure setup(First commit) --- component/tiered_storage/tiered_storage.go | 28 +++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 97f6c7b4c..65ff097d2 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -28,6 +28,7 @@ package tiered_storage import ( "context" "fmt" + "sync" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -46,6 +47,27 @@ import ( // Common structure for Component type TieredStorage struct { internal.BaseComponent + fileMap map[string]*FileNode + lruQueue *LRUQueue + mu sync.Mutex +} + +// define a file node structure to hold file related information +type FileNode struct { + name string + size uint64 + prev *FileNode + next *FileNode + cloudBacked bool + // Add more attributes as needed, e.g., last accessed time, etc. +} + +// Add more attributes as needed, e.g., last accessed time, etc. +type LRUQueue struct { + head *FileNode + tail *FileNode + maxSize uint64 //figure this out later based on config or some heuristics + currentSize uint64 } // Structure defining your config parameters @@ -53,7 +75,11 @@ type TieredStorageOptions struct { // e.g. var1 uint32 `config:"var1"` } -const compName = "tiered_storage" +const ( + compName = "tiered_storage" + defaultMaxEviction = 000000 //placeholder until we figure out + +) // Verification to check satisfaction criteria with Component Interface var _ internal.Component = &TieredStorage{} From f67f1d705e9c2bbab44a3391da67aa3cfbf8961d Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Wed, 3 Jun 2026 10:42:11 -0600 Subject: [PATCH 04/89] Initial attempt at openFile using design --- component/tiered_storage/tiered_storage.go | 56 +++++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 65ff097d2..cad34c085 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -28,7 +28,7 @@ package tiered_storage import ( "context" "fmt" - "sync" + "os" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -49,7 +49,11 @@ type TieredStorage struct { internal.BaseComponent fileMap map[string]*FileNode lruQueue *LRUQueue - mu sync.Mutex + + //use LockMap instead of mutex to allow parallel access to different files + fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) + tmpPath string // uses os.Separator (filepath.Join) + } // define a file node structure to hold file related information @@ -178,7 +182,55 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { return nil } +// First Function to work on!! +// OpenFile: Makes the file available in the local cache for further file operations. func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error) { + // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released + flock := c.fileLocks.Get(options.Name) + flock.Lock() + defer flock.Unlock() + + // Check if file exists in local cache, make sure to consider if file is not found + info, err := os.Stat(c.tmpPath + options.Name) + + //File exists in local cache + if err == nil { + //Read from local disk, create file node and add to file map + node := &FileNode{ + name: options.Name, + size: uint64(info.Size()), + cloudBacked: false, + } + c.fileMap[options.Name] = node + } else { + //Check if it exists in cloud, if yes create local copy, nope then we return error + info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) + if err != nil { + // file does not exist in cloud, return error + return nil, fmt.Errorf("file not found") + } + // file exists in cloud, create local copy (name doesn't matter)and add to file map + localCopyNode := &FileNode{ + name: options.Name, + size: uint64(info.Size), + cloudBacked: true, + } + c.fileMap[options.Name] = localCopyNode + + } + + // If not, check if file exists in cloud storage + + // If exists in cloud storage, fetch the file to local cache and return handle + // If not exists in cloud storage, return error (file not found) + + // create handle and record openFileOptions for later + handle := handlemap.NewHandle(options.Name) + handle.SetValue("openFileOptions", openFileOptions{flags: options.Flags, fMode: options.Mode}) + if options.Flags&os.O_APPEND != 0 { + handle.Flags.Set(handlemap.HandleOpenedAppend) + } + return nil, nil } From 0b27908663d53e8636c8dab3ec99d68298e9df3f Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Wed, 3 Jun 2026 17:08:55 -0600 Subject: [PATCH 05/89] Rough Design of isOverLocalLimit and openFileHelper, logic added to OpenFile --- component/tiered_storage/tiered_storage.go | 97 +++++++++++++++++++--- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index cad34c085..479a8f068 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -29,6 +29,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -54,6 +55,7 @@ type TieredStorage struct { fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) tmpPath string // uses os.Separator (filepath.Join) + maxCacheSize float64 } // define a file node structure to hold file related information @@ -190,10 +192,8 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H flock.Lock() defer flock.Unlock() - // Check if file exists in local cache, make sure to consider if file is not found + // Check if file exists in local cache, otherwise check cloud info, err := os.Stat(c.tmpPath + options.Name) - - //File exists in local cache if err == nil { //Read from local disk, create file node and add to file map node := &FileNode{ @@ -215,23 +215,98 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H size: uint64(info.Size), cloudBacked: true, } + // check if we are over the local cache limit + if c.isOverLocalLimit(uint64(info.Size), options.Name, "open") { + // we are over the local cache limit, return error for now, later we can add eviction logic here + //some sort of eviction logic here + } + //download it to the local cache and add to file map + err = c.openFileHelper(options) + if err != nil { + return nil, err + } c.fileMap[options.Name] = localCopyNode } - - // If not, check if file exists in cloud storage - - // If exists in cloud storage, fetch the file to local cache and return handle - // If not exists in cloud storage, return error (file not found) - // create handle and record openFileOptions for later handle := handlemap.NewHandle(options.Name) - handle.SetValue("openFileOptions", openFileOptions{flags: options.Flags, fMode: options.Mode}) if options.Flags&os.O_APPEND != 0 { handle.Flags.Set(handlemap.HandleOpenedAppend) } - return nil, nil + //increase handle count + flock.Inc() + + return handle, nil +} + +// openFileHelper : function to download copy from cloud and add to local cache +func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { + + //create folder if not exists, wait check what 0755 does + localPath := filepath.Join(c.tmpPath, options.Name) + err := os.MkdirAll(filepath.Dir(localPath), 0755) + if err != nil { + return err + } + //Open temporary download handle to the local file path + localFileHandle, err := common.OpenFile( + localPath, + os.O_CREATE|os.O_TRUNC|os.O_RDWR, + options.Mode, + ) + if err != nil { + return err + } + //Download + err = c.NextComponent().CopyToFile(internal.CopyToFileOptions{ + Name: options.Name, + Offset: 0, + Count: 0, + File: localFileHandle, + }) + if err != nil { + localFileHandle.Close() + os.Remove(localPath) + return err + } + localFileHandle.Close() + return nil +} + +// rough rough rough implementation of checking limit of cache, +// need to figure out eviction and other details before finalizing +func (c *TieredStorage) isOverLocalLimit( + newFileSize uint64, + fileName string, + requestType string, +) bool { + + //find ExistingSize of file if exists + existingSize := uint64(0) + if node, ok := c.fileMap[fileName]; ok { + existingSize = node.size + } + + addedFileSize := newFileSize - existingSize + + //if we didn't modify the size of the file then + if addedFileSize <= 0 { + return false + } + + //get current cache size + currSize, err := common.GetUsage(c.tmpPath) + if err != nil { + log.Err("FileCache::IsOverLocalLimit : failed to get current cache size [%v]", err) + return false + } + + if uint(currSize)+uint(addedFileSize) > uint(c.maxCacheSize) { + //should include some error message + return true + } + return false } func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int, error) { From 554a9b0cea7dc42226b11ba92502120b0840be23 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 4 Jun 2026 14:22:25 -0600 Subject: [PATCH 06/89] More structured design flow for OpenFile and setup of test file, first OpenFile test pass --- component/tiered_storage/tiered_storage.go | 52 ++++-- .../tiered_storage/tiered_storage_test.go | 169 ++++++++++++++++++ 2 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 component/tiered_storage/tiered_storage_test.go diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 479a8f068..28589bda6 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -30,6 +30,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -55,6 +56,9 @@ type TieredStorage struct { fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) tmpPath string // uses os.Separator (filepath.Join) + // Still need mutex to protect fileMap and lruQueue + mu sync.Mutex + maxCacheSize float64 } @@ -192,8 +196,20 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H flock.Lock() defer flock.Unlock() - // Check if file exists in local cache, otherwise check cloud - info, err := os.Stat(c.tmpPath + options.Name) + //1. Initial Check Map + c.mu.Lock() + _, exists := c.fileMap[options.Name] + c.mu.Unlock() + if exists { + handle := handlemap.NewHandle(options.Name) + if options.Flags&os.O_APPEND != 0 { + handle.Flags.Set(handlemap.HandleOpenedAppend) + } + flock.Inc() + return handle, nil + } + //2. Check if File exists in Disk, if not check cloud + info, err := os.Stat(filepath.Join(c.tmpPath, options.Name)) if err == nil { //Read from local disk, create file node and add to file map node := &FileNode{ @@ -201,13 +217,15 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H size: uint64(info.Size()), cloudBacked: false, } + c.mu.Lock() c.fileMap[options.Name] = node + c.mu.Unlock() } else { - //Check if it exists in cloud, if yes create local copy, nope then we return error + //3. Check if File exists in Cloud info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) if err != nil { // file does not exist in cloud, return error - return nil, fmt.Errorf("file not found") + return nil, fmt.Errorf("file not found in cloud") } // file exists in cloud, create local copy (name doesn't matter)and add to file map localCopyNode := &FileNode{ @@ -217,16 +235,17 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H } // check if we are over the local cache limit if c.isOverLocalLimit(uint64(info.Size), options.Name, "open") { - // we are over the local cache limit, return error for now, later we can add eviction logic here - //some sort of eviction logic here + // we are over the local cache limit, return error for now, + return nil, fmt.Errorf("cache limit exceeded, cannot open file") } //download it to the local cache and add to file map err = c.openFileHelper(options) if err != nil { return nil, err } + c.mu.Lock() c.fileMap[options.Name] = localCopyNode - + c.mu.Unlock() } // create handle and record openFileOptions for later handle := handlemap.NewHandle(options.Name) @@ -270,6 +289,8 @@ func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { os.Remove(localPath) return err } + //some sort of mode handling + localFileHandle.Close() return nil } @@ -282,11 +303,18 @@ func (c *TieredStorage) isOverLocalLimit( requestType string, ) bool { + if c.maxCacheSize == 0 { + // if maxCacheSize is 0, it means there is no limit on local cache size, so we can return false + return false + } + //find ExistingSize of file if exists existingSize := uint64(0) + c.mu.Lock() if node, ok := c.fileMap[fileName]; ok { existingSize = node.size } + c.mu.Unlock() addedFileSize := newFileSize - existingSize @@ -298,11 +326,11 @@ func (c *TieredStorage) isOverLocalLimit( //get current cache size currSize, err := common.GetUsage(c.tmpPath) if err != nil { - log.Err("FileCache::IsOverLocalLimit : failed to get current cache size [%v]", err) + log.Err("TieredStorage::IsOverLocalLimit : failed to get current cache size [%v]", err) return false } - if uint(currSize)+uint(addedFileSize) > uint(c.maxCacheSize) { + if float64(currSize)+float64(addedFileSize) > (c.maxCacheSize + 4096) { //should include some error message return true } @@ -376,7 +404,11 @@ func (c *TieredStorage) StatFs() (*common.Statfs_t, bool, error) { // Pipeline will call this method to create your object, initialize your variables here // << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> func NewTieredStorageComponent() internal.Component { - comp := &TieredStorage{} + comp := &TieredStorage{ + fileMap: make(map[string]*FileNode), + lruQueue: &LRUQueue{}, + fileLocks: common.NewLockMap(), + } comp.SetName(compName) return comp } diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go new file mode 100644 index 000000000..ee827b5f2 --- /dev/null +++ b/component/tiered_storage/tiered_storage_test.go @@ -0,0 +1,169 @@ +package tiered_storage + +import ( + "context" + "crypto/rand" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/config" + "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/component/loopback" + "github.com/Seagate/cloudfuse/internal" + "go.uber.org/mock/gomock" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +var home_dir, _ = os.UserHomeDir() + +type tieredStorageTestSuite struct { + suite.Suite + assert *assert.Assertions + tieredStorage *TieredStorage + loopback internal.Component + cache_path string // uses os.Separator (filepath.Join) + fake_storage_path string // uses os.Separator (filepath.Join) + useMock bool + mockCtrl *gomock.Controller + mock *internal.MockComponent +} + +func newLoopbackFS(cachePath string) internal.Component { + loopback := loopback.NewLoopbackFSComponent() + _ = loopback.Configure(true) + return loopback +} + +func newTestTieredStorage(next internal.Component) *TieredStorage { + + tieredStorage := NewTieredStorageComponent() + tieredStorage.SetNextComponent(next) + err := tieredStorage.Configure(true) + if err != nil { + panic(fmt.Sprintf("Unable to configure tiered storage: %v", err)) + } + return tieredStorage.(*TieredStorage) +} + +func randomString(length int) string { + b := make([]byte, length) + _, err := rand.Read(b) + if err != nil { + panic(err) + } + return fmt.Sprintf("%x", b)[:length] +} + +func (suite *tieredStorageTestSuite) SetupTest() { + err := log.SetDefaultLogger("silent", common.LogConfig{Level: common.ELogLevel.LOG_DEBUG()}) + if err != nil { + panic(fmt.Sprintf("Unable to set silent logger as default: %v", err)) + } + rand := randomString(8) + suite.cache_path = filepath.Join(home_dir, "file_cache"+rand) + suite.fake_storage_path = filepath.Join(home_dir, "fake_storage"+rand) + defaultConfig := fmt.Sprintf( + "file_cache:\n path: %s\n offload-io: true\n\nloopbackfs:\n path: %s", + suite.cache_path, + suite.fake_storage_path, + ) + suite.useMock = false + log.Debug("%s", defaultConfig) + + // Delete the temp directories created + err = os.RemoveAll(suite.cache_path) + if err != nil { + fmt.Printf( + "fileCacheTestSuite::SetupTest : os.RemoveAll(%s) failed [%v]\n", + suite.cache_path, + err, + ) + } + err = os.RemoveAll(suite.fake_storage_path) + if err != nil { + fmt.Printf( + "fileCacheTestSuite::SetupTest : os.RemoveAll(%s) failed [%v]\n", + suite.fake_storage_path, + err, + ) + } + suite.setupTestHelper(defaultConfig) +} + +func (suite *tieredStorageTestSuite) setupTestHelper(configuration string) { + suite.assert = assert.New(suite.T()) + + err := config.ReadConfigFromReader(strings.NewReader(configuration)) + suite.assert.NoError(err) + if suite.useMock { + suite.mockCtrl = gomock.NewController(suite.T()) + suite.mock = internal.NewMockComponent(suite.mockCtrl) + suite.tieredStorage = newTestTieredStorage(suite.mock) + // always simulate being offline + suite.mock.EXPECT().CloudConnected().AnyTimes().Return(false) + } else { + suite.loopback = newLoopbackFS(suite.fake_storage_path) + suite.tieredStorage = newTestTieredStorage(suite.loopback) + err = suite.loopback.Start(context.Background()) + suite.assert.NoError(err) + } + err = suite.tieredStorage.Start(context.Background()) + if err != nil { + panic(fmt.Sprintf("Unable to start tiered storage [%s]", err.Error())) + } + +} + +func (suite *tieredStorageTestSuite) cleanupTest() { + err := suite.tieredStorage.Stop() + if err != nil { + panic(fmt.Sprintf("Unable to stop tiered storage [%s]", err.Error())) + } + if suite.useMock { + suite.mockCtrl.Finish() + } else { + err = suite.loopback.Stop() + suite.assert.NoError(err) + } + + // Delete the temp directories created + err = os.RemoveAll(suite.cache_path) + suite.assert.NoError(err) + err = os.RemoveAll(suite.fake_storage_path) + suite.assert.NoError(err) +} + +func (suite *tieredStorageTestSuite) TestOpenFileNotInCache() { + defer suite.cleanupTest() + path := "file7" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(err) + suite.assert.Equal(path, handle.Path) + + // Verify it was now downloaded to the local tiered storage cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) +} From d77b69ff9159158f5dba408d5f43a6610f95572d Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 4 Jun 2026 16:30:42 -0600 Subject: [PATCH 07/89] Included function to make tests run, initial implementation of CreateFile --- component/tiered_storage/tiered_storage.go | 47 ++++++++++++++++++- .../tiered_storage/tiered_storage_test.go | 4 ++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 28589bda6..f551bd55f 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -178,17 +178,60 @@ func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { } // File operations +// Second Funtion to work on!! func (c *TieredStorage) CreateFile( options internal.CreateFileOptions, ) (*handlemap.Handle, error) { - return nil, nil + flock := c.fileLocks.Get(options.Name) + flock.Lock() + defer flock.Unlock() + + if c.isOverLocalLimit(0, options.Name, "create") { + return nil, fmt.Errorf("cache limit exceeded, cannot create file") + //eventually put a eviction here + } + + //Create the file in the local cache, we will ignore the create empty and cloud stuff for now + localPath := filepath.Join(c.tmpPath, options.Name) + err := os.MkdirAll(filepath.Dir(localPath), 0755) + + if err != nil { + return nil, err + } + + //Open local file + localFile, err := common.OpenFile( + localPath, + os.O_CREATE|os.O_TRUNC|os.O_RDWR, + options.Mode, + ) + if err != nil { + return nil, err + } + + //Add file node to file map with cloudBacked as false + node := &FileNode{ + name: options.Name, + size: uint64(0), + cloudBacked: false, + } + c.mu.Lock() + c.fileMap[options.Name] = node + c.mu.Unlock() + + //create handle + handle := handlemap.NewHandle(options.Name) + handle.SetFileObject(localFile) + + flock.Inc() + + return handle, nil } func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { return nil } -// First Function to work on!! // OpenFile: Makes the file available in the local cache for further file operations. func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error) { // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index ee827b5f2..b944ac9ac 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "testing" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -167,3 +168,6 @@ func (suite *tieredStorageTestSuite) TestOpenFileNotInCache() { // Verify it was now downloaded to the local tiered storage cache suite.assert.FileExists(filepath.Join(suite.cache_path, path)) } +func TestTieredStorageTestSuite(t *testing.T) { + suite.Run(t, new(tieredStorageTestSuite)) +} From ddd1a1be757a2721b764bb6b74767f58ebba7df6 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 5 Jun 2026 11:16:49 -0600 Subject: [PATCH 08/89] OpenFile now actually opens the file instead of just returning handle with no file attatched, also added some config path --- component/tiered_storage/tiered_storage.go | 114 +++++++++++------- .../tiered_storage/tiered_storage_test.go | 6 +- 2 files changed, 71 insertions(+), 49 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index f551bd55f..fb404f59e 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -83,6 +83,7 @@ type LRUQueue struct { // Structure defining your config parameters type TieredStorageOptions struct { // e.g. var1 uint32 `config:"var1"` + TmpPath string `config:"path" yaml:"path,omitempty"` } const ( @@ -138,6 +139,11 @@ func (c *TieredStorage) Configure(_ bool) error { return fmt.Errorf("TieredStorage: config error [invalid config attributes]") } // Extract values from 'conf' and store them as you wish here + // CLAUDE GENERATED HERE, CAUSE I HAD NO CLUE + c.tmpPath = filepath.Clean(common.ExpandPath(conf.TmpPath)) + if c.tmpPath == "" || c.tmpPath == "." { + return fmt.Errorf("TieredStorage: path not set in config") + } return nil } @@ -244,54 +250,66 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H _, exists := c.fileMap[options.Name] c.mu.Unlock() if exists { - handle := handlemap.NewHandle(options.Name) - if options.Flags&os.O_APPEND != 0 { - handle.Flags.Set(handlemap.HandleOpenedAppend) - } - flock.Inc() - return handle, nil - } - //2. Check if File exists in Disk, if not check cloud - info, err := os.Stat(filepath.Join(c.tmpPath, options.Name)) - if err == nil { - //Read from local disk, create file node and add to file map - node := &FileNode{ - name: options.Name, - size: uint64(info.Size()), - cloudBacked: false, - } - c.mu.Lock() - c.fileMap[options.Name] = node - c.mu.Unlock() + //skip to opening file since it should already be in local cache } else { - //3. Check if File exists in Cloud - info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) - if err != nil { - // file does not exist in cloud, return error - return nil, fmt.Errorf("file not found in cloud") - } - // file exists in cloud, create local copy (name doesn't matter)and add to file map - localCopyNode := &FileNode{ - name: options.Name, - size: uint64(info.Size), - cloudBacked: true, - } - // check if we are over the local cache limit - if c.isOverLocalLimit(uint64(info.Size), options.Name, "open") { - // we are over the local cache limit, return error for now, - return nil, fmt.Errorf("cache limit exceeded, cannot open file") + //2. Check if File exists in Disk, if not check cloud + info, err := os.Stat(filepath.Join(c.tmpPath, options.Name)) + if err == nil { + //Read from local disk, create file node and add to file map + node := &FileNode{ + name: options.Name, + size: uint64(info.Size()), + cloudBacked: false, + } + c.mu.Lock() + c.fileMap[options.Name] = node + c.mu.Unlock() + } else { + //3. Check if File exists in Cloud + info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) + if err != nil { + // file does not exist in cloud, return error + return nil, fmt.Errorf("file not found in cloud") + } + // file exists in cloud, create local copy (name doesn't matter)and add to file map + localCopyNode := &FileNode{ + name: options.Name, + size: uint64(info.Size), + cloudBacked: true, + } + // check if we are over the local cache limit + if c.isOverLocalLimit(uint64(info.Size), options.Name, "open") { + // we are over the local cache limit, return error for now, + return nil, fmt.Errorf("cache limit exceeded, cannot open file") + } + //download it to the local cache and add to file map + err = c.openFileHelper(options) + if err != nil { + return nil, err + } + c.mu.Lock() + c.fileMap[options.Name] = localCopyNode + c.mu.Unlock() } - //download it to the local cache and add to file map - err = c.openFileHelper(options) - if err != nil { - return nil, err - } - c.mu.Lock() - c.fileMap[options.Name] = localCopyNode - c.mu.Unlock() + } - // create handle and record openFileOptions for later + + //At this point the file should be in the local cache, so we can proceed to open it + + //Open the file in the local cache + localPath := filepath.Join(c.tmpPath, options.Name) + localFile, err := common.OpenFile( + localPath, + os.O_RDWR, + options.Mode, + ) + if err != nil { + return nil, err + } + + // Create handle and attach file object to it handle := handlemap.NewHandle(options.Name) + handle.SetFileObject(localFile) if options.Flags&os.O_APPEND != 0 { handle.Flags.Set(handlemap.HandleOpenedAppend) } @@ -333,7 +351,6 @@ func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { return err } //some sort of mode handling - localFileHandle.Close() return nil } @@ -385,6 +402,11 @@ func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int } func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { + //1.Check if not opened + + //2.Get the file opbject + //3. Calculate the new size + return 0, nil } @@ -419,7 +441,7 @@ func (c *TieredStorage) ReadLink(options internal.ReadLinkOptions) (string, erro // Filesystem level operations func (c *TieredStorage) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { - return &internal.ObjAttr{}, nil + return c.NextComponent().GetAttr(options) } func (c *TieredStorage) Chmod(options internal.ChmodOptions) error { diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index b944ac9ac..804547b6f 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -69,7 +69,7 @@ func (suite *tieredStorageTestSuite) SetupTest() { suite.cache_path = filepath.Join(home_dir, "file_cache"+rand) suite.fake_storage_path = filepath.Join(home_dir, "fake_storage"+rand) defaultConfig := fmt.Sprintf( - "file_cache:\n path: %s\n offload-io: true\n\nloopbackfs:\n path: %s", + "tiered_storage:\n path: %s\n offload-io: true\n\nloopbackfs:\n path: %s", suite.cache_path, suite.fake_storage_path, ) @@ -80,7 +80,7 @@ func (suite *tieredStorageTestSuite) SetupTest() { err = os.RemoveAll(suite.cache_path) if err != nil { fmt.Printf( - "fileCacheTestSuite::SetupTest : os.RemoveAll(%s) failed [%v]\n", + "tieredStorageTestSuite::SetupTest : os.RemoveAll(%s) failed [%v]\n", suite.cache_path, err, ) @@ -88,7 +88,7 @@ func (suite *tieredStorageTestSuite) SetupTest() { err = os.RemoveAll(suite.fake_storage_path) if err != nil { fmt.Printf( - "fileCacheTestSuite::SetupTest : os.RemoveAll(%s) failed [%v]\n", + "tieredStorageTestSuite::SetupTest : os.RemoveAll(%s) failed [%v]\n", suite.fake_storage_path, err, ) From 3218f914e190b933598ca70cc9915feb215b30b5 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 5 Jun 2026 13:20:36 -0600 Subject: [PATCH 09/89] Added WriteFile and dirty handle setters --- component/tiered_storage/tiered_storage.go | 70 ++++++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index fb404f59e..aeae1122e 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -402,12 +402,48 @@ func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int } func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { - //1.Check if not opened + //1.Get the file opbject + f := options.Handle.GetFileObject() + if f == nil { + return 0, fmt.Errorf("invalid file handle") + } - //2.Get the file opbject - //3. Calculate the new size + //2. Check if exceeds limits + newSize := options.Offset + int64(len(options.Data)) + if c.isOverLocalLimit(uint64(newSize), options.Handle.Path, "write") { + return 0, fmt.Errorf("cache limit exceeded, cannot write to file") + //eventually put eviction here + } - return 0, nil + //3. Decide where to write in file + var bytesWritten int + var err error + if options.Handle.Flags.IsSet(handlemap.HandleOpenedAppend) { + //write to end of file, standard + bytesWritten, err = f.Write(options.Data) + } else { + //write to specific offset, need to use WriteAt + bytesWritten, err = f.WriteAt(options.Data, options.Offset) + } + + //4. Mark file as dirty for release later + if err == nil { + c.setHandleDirty(options.Handle) + //update file node size in file map + c.mu.Lock() + if node, ok := c.fileMap[options.Handle.Path]; ok { + node.size = uint64(newSize) + } + c.mu.Unlock() + } else { + log.Err( + "TieredStorage::WriteFile : failed to write %s [%s]", + options.Handle.Path, + err.Error(), + ) + } + + return bytesWritten, err } func (c *TieredStorage) SyncFile(options internal.SyncFileOptions) error { @@ -439,6 +475,32 @@ func (c *TieredStorage) ReadLink(options internal.ReadLinkOptions) (string, erro return "", nil } +// Dirty Handle Operations +func (c *TieredStorage) setHandleDirty(handle *handlemap.Handle) { + handle.Lock() + alreadyDirty := handle.Dirty() + if !alreadyDirty { + handle.Flags.Set(handlemap.HandleFlagDirty) + } + handle.Unlock() + if !alreadyDirty { + c.fileLocks.Get(handle.Path).IncDirty() + } +} + +// setter +func (c *TieredStorage) clearHandleDirty(handle *handlemap.Handle) { + handle.Lock() + wasDirty := handle.Dirty() + if wasDirty { + handle.Flags.Clear(handlemap.HandleFlagDirty) + } + handle.Unlock() + if wasDirty { + c.fileLocks.Get(handle.Path).DecDirty() + } +} + // Filesystem level operations func (c *TieredStorage) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { return c.NextComponent().GetAttr(options) From de9b851f5e35c5c78f5a2b80ea08a3f7598152e0 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 5 Jun 2026 15:54:52 -0600 Subject: [PATCH 10/89] Added tests for open, write, create --- component/tiered_storage/tiered_storage.go | 8 +- .../tiered_storage/tiered_storage_test.go | 82 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index aeae1122e..c5eaee630 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -31,6 +31,7 @@ import ( "os" "path/filepath" "sync" + "syscall" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -229,6 +230,9 @@ func (c *TieredStorage) CreateFile( handle := handlemap.NewHandle(options.Name) handle.SetFileObject(localFile) + //Mark as dirty because the cloud doesn't know about it + c.setHandleDirty(handle) + flock.Inc() return handle, nil @@ -405,13 +409,13 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro //1.Get the file opbject f := options.Handle.GetFileObject() if f == nil { - return 0, fmt.Errorf("invalid file handle") + return 0, syscall.EBADF } //2. Check if exceeds limits newSize := options.Offset + int64(len(options.Data)) if c.isOverLocalLimit(uint64(newSize), options.Handle.Path, "write") { - return 0, fmt.Errorf("cache limit exceeded, cannot write to file") + return 0, syscall.ENOSPC //eventually put eviction here } diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 804547b6f..d987d4b25 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" "github.com/Seagate/cloudfuse/common" @@ -14,6 +15,7 @@ import ( "github.com/Seagate/cloudfuse/common/log" "github.com/Seagate/cloudfuse/component/loopback" "github.com/Seagate/cloudfuse/internal" + "github.com/Seagate/cloudfuse/internal/handlemap" "go.uber.org/mock/gomock" "github.com/stretchr/testify/assert" @@ -139,6 +141,8 @@ func (suite *tieredStorageTestSuite) cleanupTest() { suite.assert.NoError(err) } +//Testing OpenFile + func (suite *tieredStorageTestSuite) TestOpenFileNotInCache() { defer suite.cleanupTest() path := "file7" @@ -168,6 +172,84 @@ func (suite *tieredStorageTestSuite) TestOpenFileNotInCache() { // Verify it was now downloaded to the local tiered storage cache suite.assert.FileExists(filepath.Join(suite.cache_path, path)) } + +func (suite *tieredStorageTestSuite) TestOpenFileInCache() { + defer suite.cleanupTest() + path := "file8" + handle, _ := suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.tieredStorage.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.NoError(err) + + // Download is required + handle, err = suite.tieredStorage.OpenFile(internal.OpenFileOptions{Name: path, Mode: 0777}) + suite.assert.NoError(err) + suite.assert.Equal(path, handle.Path) + suite.assert.False(handle.Dirty()) + + // File should exist in cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) +} + +//Testing WriteFile + +func (suite *tieredStorageTestSuite) TestWriteFile() { + defer suite.cleanupTest() + path := "file9" + handle, _ := suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + handle.Flags.Clear( + handlemap.HandleFlagDirty, + ) // Technically create file will mark it as dirty, we just want to check write file updates the dirty flag, so temporarily set this to false + testData := "test data" + data := []byte(testData) + length, err := suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + + suite.assert.NoError(err) + suite.assert.Equal(len(data), length) + // Check that the local cache updated with data + d, _ := os.ReadFile(filepath.Join(suite.cache_path, path)) + suite.assert.Equal(data, d) + suite.assert.True(handle.Dirty()) +} + +func (suite *tieredStorageTestSuite) TestWriteFileErrorBadFd() { + defer suite.cleanupTest() + // Setup + file := "file20" + //bad handle + handle := handlemap.NewHandle(file) + bytesWrittength, err := suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle}, + ) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.EBADF, err) + suite.assert.Equal(0, bytesWrittength) +} + +// Testing Create File +func (suite *tieredStorageTestSuite) TestCreateFile() { + defer suite.cleanupTest() + // Default is to not create empty files on create file to support immutable storage. + path := "file12" + options := internal.CreateFileOptions{Name: path} + f, err := suite.tieredStorage.CreateFile(options) + + suite.assert.NoError(err) + suite.assert.True(f.Dirty()) // Handle should be dirty since it was not created in cloud storage + + // Path should be added to the file cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + // Path should not be in fake storage + suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, path)) +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 598371bc400bf13e0eb23a132557ed777f4d20e3 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 5 Jun 2026 16:55:16 -0600 Subject: [PATCH 11/89] Resolved most comments from initial PR, still have to work on error control --- component/tiered_storage/tiered_storage.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index c5eaee630..25c26669e 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -145,6 +145,12 @@ func (c *TieredStorage) Configure(_ bool) error { if c.tmpPath == "" || c.tmpPath == "." { return fmt.Errorf("TieredStorage: path not set in config") } + err = os.MkdirAll(c.tmpPath, 0755) + + if err != nil { + log.Err("TieredStorage::Configure : failed to create tmp path %s [%v]", c.tmpPath, err) + return fmt.Errorf("TieredStorage: failed to create tmp path: %w", err) + } return nil } @@ -326,7 +332,6 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H // openFileHelper : function to download copy from cloud and add to local cache func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { - //create folder if not exists, wait check what 0755 does localPath := filepath.Join(c.tmpPath, options.Name) err := os.MkdirAll(filepath.Dir(localPath), 0755) @@ -342,6 +347,8 @@ func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { if err != nil { return err } + defer localFileHandle.Close() + //Download err = c.NextComponent().CopyToFile(internal.CopyToFileOptions{ Name: options.Name, @@ -351,11 +358,10 @@ func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { }) if err != nil { localFileHandle.Close() - os.Remove(localPath) + _ = os.Remove(localPath) return err } //some sort of mode handling - localFileHandle.Close() return nil } From 3ed9612a44d80642fe42ba3ebe0def9b844af9fa Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 8 Jun 2026 11:16:29 -0600 Subject: [PATCH 12/89] Added OpenFile with O_create case --- component/tiered_storage/tiered_storage.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 25c26669e..14be10b43 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -191,7 +191,6 @@ func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { } // File operations -// Second Funtion to work on!! func (c *TieredStorage) CreateFile( options internal.CreateFileOptions, ) (*handlemap.Handle, error) { @@ -255,6 +254,17 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H flock.Lock() defer flock.Unlock() + //Case 1: OpenFile with O_Create + if options.Flags&os.O_CREATE != 0 { + handle, err := c.CreateFile( + internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, + ) + if err != nil { + return nil, err + } + return handle, nil + } + //1. Initial Check Map c.mu.Lock() _, exists := c.fileMap[options.Name] @@ -279,7 +289,8 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) if err != nil { // file does not exist in cloud, return error - return nil, fmt.Errorf("file not found in cloud") + log.Err("TieredStorage::OpenFile : File Does not exist in cloud") + return nil, err } // file exists in cloud, create local copy (name doesn't matter)and add to file map localCopyNode := &FileNode{ From 55f0dfa1fd3b3307defd469aab8b6a1e3f9514fc Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 8 Jun 2026 14:30:03 -0600 Subject: [PATCH 13/89] Fixed O_Create and modified CreateFile (added unlocked internal function), added tests for OpenFile O_Create --- component/tiered_storage/tiered_storage.go | 44 +++++++---- .../tiered_storage/tiered_storage_test.go | 77 ++++++++++++++++++- 2 files changed, 107 insertions(+), 14 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 14be10b43..39b98a37f 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -191,13 +191,10 @@ func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { } // File operations -func (c *TieredStorage) CreateFile( + +func (c *TieredStorage) createFileUnlocked( options internal.CreateFileOptions, ) (*handlemap.Handle, error) { - flock := c.fileLocks.Get(options.Name) - flock.Lock() - defer flock.Unlock() - if c.isOverLocalLimit(0, options.Name, "create") { return nil, fmt.Errorf("cache limit exceeded, cannot create file") //eventually put a eviction here @@ -214,7 +211,7 @@ func (c *TieredStorage) CreateFile( //Open local file localFile, err := common.OpenFile( localPath, - os.O_CREATE|os.O_TRUNC|os.O_RDWR, + os.O_CREATE|os.O_RDWR, options.Mode, ) if err != nil { @@ -238,8 +235,21 @@ func (c *TieredStorage) CreateFile( //Mark as dirty because the cloud doesn't know about it c.setHandleDirty(handle) - flock.Inc() + return handle, nil + +} +func (c *TieredStorage) CreateFile( + options internal.CreateFileOptions, +) (*handlemap.Handle, error) { + flock := c.fileLocks.Get(options.Name) + flock.Lock() + defer flock.Unlock() + handle, err := c.createFileUnlocked(options) + if err != nil { + return nil, err + } + flock.Inc() return handle, nil } @@ -256,13 +266,21 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H //Case 1: OpenFile with O_Create if options.Flags&os.O_CREATE != 0 { - handle, err := c.CreateFile( - internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, - ) - if err != nil { - return nil, err + //Check if file first exists, then proceed + c.mu.Lock() + _, exists := c.fileMap[options.Name] + c.mu.Unlock() + if exists { + } else { + handle, err := c.createFileUnlocked( + internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, + ) + if err != nil { + return nil, err + } + flock.Inc() + return handle, nil } - return handle, nil } //1. Initial Check Map diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index d987d4b25..2a5cd558f 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -196,11 +196,86 @@ func (suite *tieredStorageTestSuite) TestOpenFileInCache() { suite.assert.FileExists(filepath.Join(suite.cache_path, path)) } +func (suite *tieredStorageTestSuite) TestOpenFileOCreate() { + defer suite.cleanupTest() + path := "file9" + handle, err := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path, handle.Path) + suite.assert.True(handle.Dirty()) + // File should exist in cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + +} + +func (suite *tieredStorageTestSuite) TestOpenFileOCreateExistsLocal() { + defer suite.cleanupTest() + path := "file10" + handle, _ := suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.tieredStorage.FlushFile(internal.FlushFileOptions{Handle: handle}) + suite.assert.NoError(err) + + // Download is required + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path, handle.Path) + suite.assert.False(handle.Dirty()) + + // File should exist in cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + + //Make sure data didn't get modified + d, err := os.ReadFile(filepath.Join(suite.cache_path, path)) + suite.assert.NoError(err) + suite.assert.Equal(data, d) + +} + +func (suite *tieredStorageTestSuite) TestOpenFileOCreateExistsCloud() { + defer suite.cleanupTest() + path := "file11" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_CREATE, + Mode: 0777, + }, + ) + suite.assert.NoError(err) + suite.assert.Equal(path, handle.Path) + + // Verify it was now downloaded to the local tiered storage cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) +} + //Testing WriteFile func (suite *tieredStorageTestSuite) TestWriteFile() { defer suite.cleanupTest() - path := "file9" + path := "file11" handle, _ := suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) handle.Flags.Clear( handlemap.HandleFlagDirty, From ffc390928769220de0a55ab567bdbfe2f80e7768 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 8 Jun 2026 16:58:04 -0600 Subject: [PATCH 14/89] Added initial ReleaseFile and uploadFile components --- component/tiered_storage/tiered_storage.go | 67 +++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 39b98a37f..56926b5b4 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -191,7 +191,6 @@ func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { } // File operations - func (c *TieredStorage) createFileUnlocked( options internal.CreateFileOptions, ) (*handlemap.Handle, error) { @@ -494,9 +493,75 @@ func (c *TieredStorage) FlushFile(options internal.FlushFileOptions) error { } func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { + // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released + flock := c.fileLocks.Get(options.Handle.Path) + flock.Lock() + defer flock.Unlock() + + //Dec Handle First + flock.Dec() + + //Check if this is the last file handle + handleCount := flock.Count() + + //it is the last handle + if handleCount == 0 { + //is file cloudbacked + c.mu.Lock() + isCloudBacked := c.fileMap[options.Handle.Path].cloudBacked + c.mu.Unlock() + if isCloudBacked { + //File was modified + if options.Handle.Dirty() { + //Upload + err := c.uploadFile(options.Handle.Path) + //Delete local file copy + c.mu.Lock() + delete(c.fileMap, options.Handle.Path) + c.mu.Unlock() + //Clean Handle + options.Handle.Cleanup() + } else { + //File was not modified + c.mu.Lock() + delete(c.fileMap, options.Handle.Path) + c.mu.Unlock() + options.Handle.Cleanup() + } + } else { + //local only then just close the file, update LRU add to queue, we will get to this later + options.Handle.Cleanup() + } + + } return nil } +func (c *TieredStorage) uploadFile(name string) error { + //get the local path + localPath := filepath.Join(c.tmpPath, name) + _, err := os.Stat(localPath) + if err != nil { + log.Err("TieredStorage::uploadFile : %s stat failed [%v]", name, err) + return err + } + + //open read-only handle/file for uploading + f, openErr := common.Open(localPath) + if openErr != nil { + log.Err("TieredStorage::uploadFile : %s open failed [%v]", name, openErr) + return openErr + } + defer f.Close() + + //upload + uploadErr := c.NextComponent().CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) + if uploadErr != nil { + log.Err("TieredStorage::uploadFile : %s upload failed [%v]", name, uploadErr) + } + return uploadErr +} + func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { return nil } From 9cef74d44e7605ac611260444f7c88b6ba95dc2c Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 9 Jun 2026 11:18:38 -0600 Subject: [PATCH 15/89] Implemented initial design of ReleaseFile and corresponding tests --- component/tiered_storage/tiered_storage.go | 24 +++- .../tiered_storage/tiered_storage_test.go | 110 ++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 56926b5b4..aa0ce8cc4 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -508,36 +508,50 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { if handleCount == 0 { //is file cloudbacked c.mu.Lock() - isCloudBacked := c.fileMap[options.Handle.Path].cloudBacked + node, _ := c.fileMap[options.Handle.Path] c.mu.Unlock() - if isCloudBacked { + + if node.cloudBacked { //File was modified if options.Handle.Dirty() { //Upload - err := c.uploadFile(options.Handle.Path) + err := c.uploadCachedFile(options.Handle.Path) + if err != nil { + log.Err( + "TieredStorage::ReleaseFile : upload failed for %s [%v]", + options.Handle.Path, + err, + ) + options.Handle.Cleanup() + return err + } //Delete local file copy + localPath := filepath.Join(c.tmpPath, options.Handle.Path) c.mu.Lock() delete(c.fileMap, options.Handle.Path) c.mu.Unlock() //Clean Handle options.Handle.Cleanup() + os.Remove(localPath) } else { //File was not modified + localPath := filepath.Join(c.tmpPath, options.Handle.Path) c.mu.Lock() delete(c.fileMap, options.Handle.Path) c.mu.Unlock() options.Handle.Cleanup() + os.Remove(localPath) + } } else { //local only then just close the file, update LRU add to queue, we will get to this later options.Handle.Cleanup() } - } return nil } -func (c *TieredStorage) uploadFile(name string) error { +func (c *TieredStorage) uploadCachedFile(name string) error { //get the local path localPath := filepath.Join(c.tmpPath, name) _, err := os.Stat(localPath) diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 2a5cd558f..779877a4b 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -325,6 +325,116 @@ func (suite *tieredStorageTestSuite) TestCreateFile() { suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, path)) } +// Testing Release File +func (suite *tieredStorageTestSuite) TestReleaseCloudNoDirtyFile() { + defer suite.cleanupTest() + path := "file13" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + err := suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, openErr := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(openErr) + + // Verify it was now downloaded to the local tiered storage cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + + //As of now, the file would be cloudbacked and exist in map + suite.tieredStorage.mu.Lock() + node, exists := suite.tieredStorage.fileMap[path] + suite.tieredStorage.mu.Unlock() + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(exists, "File should be tracked in the fileMap") + + //File should be "cloudBacked" and not dirty so on release the file should be deleted from local and the handle clean + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + _, err = os.Stat(filepath.Join(suite.cache_path, path)) + suite.assert.True(os.IsNotExist(err), "File should be deleted from cache after release") + +} + +func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { + defer suite.cleanupTest() + path := "file13" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, openErr := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(openErr) + + // Verify it was now downloaded to the local tiered storage cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + + //As of now, the file would be cloudbacked and exist in map + suite.tieredStorage.mu.Lock() + node, exists := suite.tieredStorage.fileMap[path] + suite.tieredStorage.mu.Unlock() + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(exists, "File should be tracked in the fileMap") + + //File should be "cloudBacked" and dirty so on release the file should be deleted from local and the handle clean + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + _, err = os.Stat(filepath.Join(suite.cache_path, path)) + suite.assert.True(os.IsNotExist(err), "File should be deleted from cache after release") + + //Must check that file by its data is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: path, RetrieveMetadata: true}) + suite.assert.NoError(err) + + //tmpFile to hold cloud data || WARNING AI SLOP BELOW, I did not write below this + tmpFile, err := os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err := os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) + +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 6b919a06027cb629150ea1c042843aa45fc5f495 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 11 Jun 2026 15:37:46 -0600 Subject: [PATCH 16/89] Implemented initial design of ReleaseFile and corresponding tests --- component/tiered_storage/tiered_storage.go | 24 +++- .../tiered_storage/tiered_storage_test.go | 110 ++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 56926b5b4..aa0ce8cc4 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -508,36 +508,50 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { if handleCount == 0 { //is file cloudbacked c.mu.Lock() - isCloudBacked := c.fileMap[options.Handle.Path].cloudBacked + node, _ := c.fileMap[options.Handle.Path] c.mu.Unlock() - if isCloudBacked { + + if node.cloudBacked { //File was modified if options.Handle.Dirty() { //Upload - err := c.uploadFile(options.Handle.Path) + err := c.uploadCachedFile(options.Handle.Path) + if err != nil { + log.Err( + "TieredStorage::ReleaseFile : upload failed for %s [%v]", + options.Handle.Path, + err, + ) + options.Handle.Cleanup() + return err + } //Delete local file copy + localPath := filepath.Join(c.tmpPath, options.Handle.Path) c.mu.Lock() delete(c.fileMap, options.Handle.Path) c.mu.Unlock() //Clean Handle options.Handle.Cleanup() + os.Remove(localPath) } else { //File was not modified + localPath := filepath.Join(c.tmpPath, options.Handle.Path) c.mu.Lock() delete(c.fileMap, options.Handle.Path) c.mu.Unlock() options.Handle.Cleanup() + os.Remove(localPath) + } } else { //local only then just close the file, update LRU add to queue, we will get to this later options.Handle.Cleanup() } - } return nil } -func (c *TieredStorage) uploadFile(name string) error { +func (c *TieredStorage) uploadCachedFile(name string) error { //get the local path localPath := filepath.Join(c.tmpPath, name) _, err := os.Stat(localPath) diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 2a5cd558f..779877a4b 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -325,6 +325,116 @@ func (suite *tieredStorageTestSuite) TestCreateFile() { suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, path)) } +// Testing Release File +func (suite *tieredStorageTestSuite) TestReleaseCloudNoDirtyFile() { + defer suite.cleanupTest() + path := "file13" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + err := suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, openErr := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(openErr) + + // Verify it was now downloaded to the local tiered storage cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + + //As of now, the file would be cloudbacked and exist in map + suite.tieredStorage.mu.Lock() + node, exists := suite.tieredStorage.fileMap[path] + suite.tieredStorage.mu.Unlock() + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(exists, "File should be tracked in the fileMap") + + //File should be "cloudBacked" and not dirty so on release the file should be deleted from local and the handle clean + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + _, err = os.Stat(filepath.Join(suite.cache_path, path)) + suite.assert.True(os.IsNotExist(err), "File should be deleted from cache after release") + +} + +func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { + defer suite.cleanupTest() + path := "file13" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, openErr := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(openErr) + + // Verify it was now downloaded to the local tiered storage cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + + //As of now, the file would be cloudbacked and exist in map + suite.tieredStorage.mu.Lock() + node, exists := suite.tieredStorage.fileMap[path] + suite.tieredStorage.mu.Unlock() + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(exists, "File should be tracked in the fileMap") + + //File should be "cloudBacked" and dirty so on release the file should be deleted from local and the handle clean + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + _, err = os.Stat(filepath.Join(suite.cache_path, path)) + suite.assert.True(os.IsNotExist(err), "File should be deleted from cache after release") + + //Must check that file by its data is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: path, RetrieveMetadata: true}) + suite.assert.NoError(err) + + //tmpFile to hold cloud data || WARNING AI SLOP BELOW, I did not write below this + tmpFile, err := os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err := os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) + +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 0b199fca8dc45a63cca9c50d1956394ff4ef74ad Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 11 Jun 2026 15:40:16 -0600 Subject: [PATCH 17/89] 2nd PR Changes --- component/tiered_storage/tiered_storage.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index aa0ce8cc4..f78886277 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -263,6 +263,8 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H flock.Lock() defer flock.Unlock() + //Go through flag cases, might need to explore O_TRUNC + //Case 1: OpenFile with O_Create if options.Flags&os.O_CREATE != 0 { //Check if file first exists, then proceed @@ -286,9 +288,9 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H c.mu.Lock() _, exists := c.fileMap[options.Name] c.mu.Unlock() - if exists { - //skip to opening file since it should already be in local cache - } else { + + //if exists skip to opening file since it should already be in local cache + if !exists { //2. Check if File exists in Disk, if not check cloud info, err := os.Stat(filepath.Join(c.tmpPath, options.Name)) if err == nil { @@ -414,7 +416,7 @@ func (c *TieredStorage) isOverLocalLimit( } c.mu.Unlock() - addedFileSize := newFileSize - existingSize + addedFileSize := int64(newFileSize) - int64(existingSize) //if we didn't modify the size of the file then if addedFileSize <= 0 { @@ -508,9 +510,17 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { if handleCount == 0 { //is file cloudbacked c.mu.Lock() - node, _ := c.fileMap[options.Handle.Path] + node, exists := c.fileMap[options.Handle.Path] c.mu.Unlock() + if !exists { + log.Err( + "TieredStorage::ReleaseFile : internal error: file %s not found in map", + options.Handle.Path, + ) + return syscall.EBADF + } + if node.cloudBacked { //File was modified if options.Handle.Dirty() { From d51bfe8d0c42bd8a200d221e6c7612d436890674 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 11 Jun 2026 15:42:02 -0600 Subject: [PATCH 18/89] Initial ReadInBuffer component --- component/tiered_storage/tiered_storage.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index aa0ce8cc4..59c8e2574 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -28,6 +28,7 @@ package tiered_storage import ( "context" "fmt" + "io" "os" "path/filepath" "sync" @@ -436,7 +437,21 @@ func (c *TieredStorage) isOverLocalLimit( } func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int, error) { - return 0, nil + f := options.Handle.GetFileObject() + if f == nil { + log.Err( + "TieredStorage::ReadInBuffer : error [couldn't find fd in handle] %s", + options.Handle.Path, + ) + return 0, syscall.EBADF + } + + n, err := f.ReadAt(options.Data, options.Offset) + // ReadAt gives an error if it reads fewer bytes than the byte array. We discard that error. + if n < len(options.Data) && err == io.EOF { + return n, nil + } + return n, err } func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { From d805f913416898187a8f17eeec39a0d7f113b8b6 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 15 Jun 2026 11:12:05 -0600 Subject: [PATCH 19/89] Added initial ReadInBuffer tests --- .../tiered_storage/tiered_storage_test.go | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 779877a4b..fadf0afab 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -391,6 +391,14 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { // Verify it was now downloaded to the local tiered storage cache suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + + // Handle should be dirty since it was not created in cloud storage + suite.assert.True(handle.Dirty()) + //As of now, the file would be cloudbacked and exist in map suite.tieredStorage.mu.Lock() node, exists := suite.tieredStorage.fileMap[path] @@ -410,6 +418,7 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { suite.assert.NoError(err) //tmpFile to hold cloud data || WARNING AI SLOP BELOW, I did not write below this + //It just checks if the data is preserved tmpFile, err := os.CreateTemp("", "cloud_verify") suite.assert.NoError(err) defer os.Remove(tmpFile.Name()) @@ -435,6 +444,49 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { } +func (suite *tieredStorageTestSuite) TestReadInBuffer() { + defer suite.cleanupTest() + // Setup + file := "file14" + + //put file in cloud abd write to it + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //Must check that file by its data is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: file, RetrieveMetadata: true}) + suite.assert.NoError(err) + + handle, _ = suite.tieredStorage.OpenFile(internal.OpenFileOptions{Name: file, Mode: 0777}) + + output := make([]byte, 9) + length, err := suite.tieredStorage.ReadInBuffer( + &internal.ReadInBufferOptions{Handle: handle, Offset: 0, Data: output}, + ) + suite.assert.NoError(err) + suite.assert.Equal(data, output) + suite.assert.Equal(len(data), length) +} + +func (suite *tieredStorageTestSuite) TestReadInBufferErrorBadFd() { + defer suite.cleanupTest() + // Setup + file := "file15" + handle := handlemap.NewHandle(file) + length, err := suite.tieredStorage.ReadInBuffer(&internal.ReadInBufferOptions{Handle: handle}) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.EBADF, err) + suite.assert.Equal(0, length) +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 432d54d569439b2137bb821584625d7ffcec0c55 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 15 Jun 2026 15:09:34 -0600 Subject: [PATCH 20/89] Initial implementation of DeleteFile, no tests yet --- component/tiered_storage/tiered_storage.go | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 59c8e2574..65acdb717 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -254,6 +254,49 @@ func (c *TieredStorage) CreateFile( } func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { + //Lock the file first + flock := c.fileLocks.Get(options.Name) + flock.Lock() + defer flock.Unlock() + + //Check file map + c.mu.Lock() + node, exists := c.fileMap[options.Name] + c.mu.Unlock() + if exists { + //local only + if !node.cloudBacked { + //delete locally + localPath := filepath.Join(c.tmpPath, options.Name) + c.mu.Lock() + delete(c.fileMap, options.Name) + c.mu.Unlock() + os.Remove(localPath) + + //Both + } else { + //delete from cloud + err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) + if err != nil { + return err + } + //delete locally + localPath := filepath.Join(c.tmpPath, options.Name) + c.mu.Lock() + delete(c.fileMap, options.Name) + c.mu.Unlock() + os.Remove(localPath) + } + + } else { + //check cloud, else return an error + err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) + if err != nil { + return syscall.ENOENT + } + return err + } + return nil } From 012bfc928592d1bd3c345d1f8c4bd25ebfdde2f9 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Wed, 24 Jun 2026 14:55:21 -0600 Subject: [PATCH 21/89] Resolved additional PR1 comments --- component/tiered_storage/tiered_storage.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index f78886277..3cb1b31f9 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -271,8 +271,7 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H c.mu.Lock() _, exists := c.fileMap[options.Name] c.mu.Unlock() - if exists { - } else { + if !exists { handle, err := c.createFileUnlocked( internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, ) @@ -294,6 +293,10 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H //2. Check if File exists in Disk, if not check cloud info, err := os.Stat(filepath.Join(c.tmpPath, options.Name)) if err == nil { + log.Warn( + "TieredStorage::OpenFile : Warning file exists locally on disk but not in tiered storage cache: %s", + options.Name, + ) //Read from local disk, create file node and add to file map node := &FileNode{ name: options.Name, @@ -308,7 +311,9 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) if err != nil { // file does not exist in cloud, return error - log.Err("TieredStorage::OpenFile : File Does not exist in cloud") + log.Err("TieredStorage::OpenFile : File Does not exist in cloud or local cache: %s", + options.Name, + ) return nil, err } // file exists in cloud, create local copy (name doesn't matter)and add to file map @@ -323,7 +328,7 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H return nil, fmt.Errorf("cache limit exceeded, cannot open file") } //download it to the local cache and add to file map - err = c.openFileHelper(options) + err = c.downloadCopyFromCloud(options) if err != nil { return nil, err } @@ -361,7 +366,7 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H } // openFileHelper : function to download copy from cloud and add to local cache -func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { +func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) error { //create folder if not exists, wait check what 0755 does localPath := filepath.Join(c.tmpPath, options.Name) err := os.MkdirAll(filepath.Dir(localPath), 0755) @@ -387,7 +392,6 @@ func (c *TieredStorage) openFileHelper(options internal.OpenFileOptions) error { File: localFileHandle, }) if err != nil { - localFileHandle.Close() _ = os.Remove(localPath) return err } From 628feaad8caa9f405b25809b302e37daa2e1d9f1 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 25 Jun 2026 16:16:02 -0600 Subject: [PATCH 22/89] Initial LRU skeleton --- component/tiered_storage/lru_policy.go | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 component/tiered_storage/lru_policy.go diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go new file mode 100644 index 000000000..15af7db5e --- /dev/null +++ b/component/tiered_storage/lru_policy.go @@ -0,0 +1,69 @@ + +package tiered_storage + +import ( + "bytes" + "encoding/gob" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/log" +) + +type lruNode struct{ + prev *lruNode + next *lruNode + name string + +} +//upload 50 files and then check + +type lruQueue struct{ + sync.Mutex + + wg sync.WaitGroup + numWorkers int + + head *lruNode + tail *lruNode + + uploadChan chan string + doneChan chan struct{} + + threshold float64 + targetRatio float64 + + +} + + +func(q *lruQueue) Touch(){} + +func(q *lruQueue) Add(){} + +func(q *lruQueue) Remove(){} + +//this will + +func worker(){} + + +func (q *lruQueue) capacityChecker() { + defer q.wg.Done() + defer close(q.uploadChan) + + for { + select { + case <-time.After(//some time idk): + // eviction + + case <-q.doneChan: + return + } + } +} \ No newline at end of file From 2ccb80fc17b91d8644140a98142f477ebc0ca644 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 26 Jun 2026 17:10:57 -0600 Subject: [PATCH 23/89] Initial LRU Functions --- component/tiered_storage/lru_policy.go | 103 ++++++++++++++++--------- 1 file changed, 65 insertions(+), 38 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 15af7db5e..6cde771d5 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -1,69 +1,96 @@ - package tiered_storage import ( - "bytes" - "encoding/gob" - "os" - "path/filepath" - "strings" "sync" - "sync/atomic" "time" - "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" ) -type lruNode struct{ - prev *lruNode +type lruNode struct { + prev *lruNode next *lruNode - name string - + name string } + //upload 50 files and then check -type lruQueue struct{ - sync.Mutex +type lruQueue struct { + mu sync.Mutex + + nodeMap sync.Map - wg sync.WaitGroup - numWorkers int + wg sync.WaitGroup + numWorkers int head *lruNode - tail *lruNode + tail *lruNode uploadChan chan string - doneChan chan struct{} + doneChan chan struct{} - threshold float64 + threshold float64 targetRatio float64 +} - +func (q *lruQueue) Touch(name string) {} + +func (q *lruQueue) Enqueue(name string) { + //Maybe have a duplicate checker + + //create node + node := &lruNode{name: name} + //Add node to map + q.nodeMap.Store(name, node) + q.mu.Lock() + defer q.mu.Unlock() + if q.head == nil { + q.head = node + q.tail = node + } else { + q.setHead(node) + } } +func (q *lruQueue) Dequeue(name string) { + log.Trace("lruPolicy::removeNode : %s", name) -func(q *lruQueue) Touch(){} + var node *lruNode = nil -func(q *lruQueue) Add(){} + val, found := q.nodeMap.LoadAndDelete(name) + if !found || val == nil { + return + } -func(q *lruQueue) Remove(){} + q.mu.Lock() + defer q.mu.Unlock() -//this will + node = val.(*lruNode) -func worker(){} + q.extractNode(node) +} + +func (q *lruQueue) setHead(node *lruNode) { + // insert node at the head + node.prev = nil + node.next = q.head + q.head.prev = node + q.head = node +} +func worker() {} func (q *lruQueue) capacityChecker() { - defer q.wg.Done() - defer close(q.uploadChan) - - for { - select { - case <-time.After(//some time idk): - // eviction - - case <-q.doneChan: - return - } - } -} \ No newline at end of file + defer q.wg.Done() + defer close(q.uploadChan) + + for { + select { + case <-time.After(2 * time.Minute): + // eviction + + case <-q.doneChan: + return + } + } +} From 63b276cf5b61dbfb19e3b7a13577906b06032963 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 29 Jun 2026 11:56:28 -0600 Subject: [PATCH 24/89] LRU eviction initial implementation --- component/tiered_storage/lru_policy.go | 94 ++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 6cde771d5..a49262f38 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -4,6 +4,7 @@ import ( "sync" "time" + "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" ) @@ -29,34 +30,43 @@ type lruQueue struct { uploadChan chan string doneChan chan struct{} + cachePath string + maxCacheSize float64 + threshold float64 targetRatio float64 } -func (q *lruQueue) Touch(name string) {} +func (q *lruQueue) Touch(name string) { + q.Enqueue(name) +} func (q *lruQueue) Enqueue(name string) { - //Maybe have a duplicate checker + //Maybe have a duplicate , that touches essentially //create node - node := &lruNode{name: name} - //Add node to map - q.nodeMap.Store(name, node) + newNode := &lruNode{name: name} + val, found := q.nodeMap.LoadOrStore(name, newNode) + node := val.(*lruNode) + q.mu.Lock() defer q.mu.Unlock() - if q.head == nil { - q.head = node - q.tail = node + + if found { + // touch + q.extractNode(node) } else { - q.setHead(node) + // brand new node — update tail if list was empty + if q.tail == nil { + q.tail = node + } } + q.setHead(node) } func (q *lruQueue) Dequeue(name string) { log.Trace("lruPolicy::removeNode : %s", name) - var node *lruNode = nil - val, found := q.nodeMap.LoadAndDelete(name) if !found || val == nil { return @@ -65,7 +75,7 @@ func (q *lruQueue) Dequeue(name string) { q.mu.Lock() defer q.mu.Unlock() - node = val.(*lruNode) + node := val.(*lruNode) q.extractNode(node) } @@ -78,6 +88,28 @@ func (q *lruQueue) setHead(node *lruNode) { q.head = node } +func (q *lruQueue) extractNode(node *lruNode) { + // remove the node from its position in the list + + // head case + if node == q.head { + q.head = node.next + } + //tail case + if node == q.tail { + q.tail = node.prev + } + + if node.next != nil { + node.next.prev = node.prev + } + if node.prev != nil { + node.prev.next = node.next + } + node.prev = nil + node.next = nil +} + func worker() {} func (q *lruQueue) capacityChecker() { @@ -88,9 +120,47 @@ func (q *lruQueue) capacityChecker() { select { case <-time.After(2 * time.Minute): // eviction + //1. check if we need eviction + curSize, err := common.GetUsage(q.cachePath) + if err != nil{ + log.Err("lruPolicy::capacityChecker : failed to get usage: %v", err) + continue + } + if curSize/q.maxCacheSize <= q.threshold{ + break + } + for curSize/q.maxCacheSize > q.targetRatio{ + if !q.eviction() { + break + } + curSize, err = common.GetUsage(q.cachePath) + if err != nil { + log.Err("lruPolicy::capacityChecker : failed to get usage after eviction: %v", err) + return + } + } + } + + //seperate function for eviction case <-q.doneChan: return } +} + +func (q *lruQueue) eviction() bool { + q.mu.Lock() + defer q.mu.Unlock() + nodeToEvict := q.tail + if nodeToEvict == nil { + return false } + + //remove node from queue and map + q.extractNode(nodeToEvict) + q.nodeMap.Delete(nodeToEvict) + + //send node to channel + q.uploadChan <- nodeToEvict.name + return true } From ca744a6a9021c3f1d8bd6a8a1c97c1945e1687ad Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 29 Jun 2026 14:03:50 -0600 Subject: [PATCH 25/89] LRU worker logic added --- component/tiered_storage/lru_policy.go | 79 ++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index a49262f38..f074e41bd 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -1,11 +1,14 @@ package tiered_storage import ( + "os" + "path/filepath" "sync" "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/internal" ) type lruNode struct { @@ -30,13 +33,38 @@ type lruQueue struct { uploadChan chan string doneChan chan struct{} - cachePath string + cachePath string maxCacheSize float64 threshold float64 targetRatio float64 } +func (q *lruQueue) StartPolicy() error { + //initialize queue + //channels + //timer + //go routines + q.wg.Add(1) + go q.capacityChecker() + + q.wg.Add(q.numWorkers) + for i := 0; i < q.numWorkers; i++ { + go worker() + } + return nil +} + +func (q *lruQueue) StopPolicy() error { + //initialize queue + //channels + //timer + //go routines + close(q.doneChan) + q.wg.Wait() + return nil +} + func (q *lruQueue) Touch(name string) { q.Enqueue(name) } @@ -110,8 +138,6 @@ func (q *lruQueue) extractNode(node *lruNode) { node.next = nil } -func worker() {} - func (q *lruQueue) capacityChecker() { defer q.wg.Done() defer close(q.uploadChan) @@ -122,30 +148,31 @@ func (q *lruQueue) capacityChecker() { // eviction //1. check if we need eviction curSize, err := common.GetUsage(q.cachePath) - if err != nil{ + if err != nil { log.Err("lruPolicy::capacityChecker : failed to get usage: %v", err) continue } - if curSize/q.maxCacheSize <= q.threshold{ + if curSize/q.maxCacheSize <= q.threshold { break } - for curSize/q.maxCacheSize > q.targetRatio{ + for curSize/q.maxCacheSize > q.targetRatio { if !q.eviction() { break } curSize, err = common.GetUsage(q.cachePath) if err != nil { - log.Err("lruPolicy::capacityChecker : failed to get usage after eviction: %v", err) - return + log.Err( + "lruPolicy::capacityChecker : failed to get usage after eviction: %v", + err, + ) + continue } } - } - - //seperate function for eviction case <-q.doneChan: return } + } } func (q *lruQueue) eviction() bool { @@ -158,9 +185,37 @@ func (q *lruQueue) eviction() bool { //remove node from queue and map q.extractNode(nodeToEvict) - q.nodeMap.Delete(nodeToEvict) + q.nodeMap.Delete(nodeToEvict.name) //send node to channel q.uploadChan <- nodeToEvict.name return true } + +func worker() { + //get the local path + localPath := filepath.Join(c.tmpPath, name) + _, err := os.Stat(localPath) + if err != nil { + log.Err("TieredStorage::uploadFile : %s stat failed [%v]", name, err) + return err + } + + //open read-only handle/file for uploading + f, openErr := common.Open(localPath) + if openErr != nil { + log.Err("TieredStorage::uploadFile : %s open failed [%v]", name, openErr) + return openErr + } + defer f.Close() + + //upload + uploadErr := c.NextComponent().CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) + if uploadErr != nil { + log.Err("TieredStorage::uploadFile : %s upload failed [%v]", name, uploadErr) + } + return uploadErr + +} + +//ok so the worker is going to have to take a file and upload to cloud From 8e31b0cd1e2422cb6ae510732cc2ce748c2362ff Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 29 Jun 2026 14:18:13 -0600 Subject: [PATCH 26/89] Initial LRU Code w Workers/Eviction logic --- component/tiered_storage/lru_policy.go | 58 +++++++++++--------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index f074e41bd..1a4059f28 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -1,14 +1,12 @@ package tiered_storage import ( - "os" - "path/filepath" + "fmt" "sync" "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" - "github.com/Seagate/cloudfuse/internal" ) type lruNode struct { @@ -38,11 +36,24 @@ type lruQueue struct { threshold float64 targetRatio float64 + + //upload function from tiered storage WIRE THIS LATER in tiered storage because we are using a function from there + upload func(name string) error } func (q *lruQueue) StartPolicy() error { + if q.upload == nil { + return fmt.Errorf("lruQueue: upload function not set") + } + if q.numWorkers <= 0 { + return fmt.Errorf("lruQueue: numWorkers must be > 0") + } //initialize queue + q.head = nil + q.tail = nil //channels + q.uploadChan = make(chan string, 1000) + q.doneChan = make(chan struct{}) //timer //go routines q.wg.Add(1) @@ -50,16 +61,13 @@ func (q *lruQueue) StartPolicy() error { q.wg.Add(q.numWorkers) for i := 0; i < q.numWorkers; i++ { - go worker() + go q.worker() } return nil } func (q *lruQueue) StopPolicy() error { - //initialize queue - //channels - //timer - //go routines + close(q.doneChan) q.wg.Wait() return nil @@ -177,7 +185,6 @@ func (q *lruQueue) capacityChecker() { func (q *lruQueue) eviction() bool { q.mu.Lock() - defer q.mu.Unlock() nodeToEvict := q.tail if nodeToEvict == nil { return false @@ -186,36 +193,19 @@ func (q *lruQueue) eviction() bool { //remove node from queue and map q.extractNode(nodeToEvict) q.nodeMap.Delete(nodeToEvict.name) + q.mu.Unlock() //send node to channel q.uploadChan <- nodeToEvict.name return true } -func worker() { - //get the local path - localPath := filepath.Join(c.tmpPath, name) - _, err := os.Stat(localPath) - if err != nil { - log.Err("TieredStorage::uploadFile : %s stat failed [%v]", name, err) - return err - } - - //open read-only handle/file for uploading - f, openErr := common.Open(localPath) - if openErr != nil { - log.Err("TieredStorage::uploadFile : %s open failed [%v]", name, openErr) - return openErr - } - defer f.Close() - - //upload - uploadErr := c.NextComponent().CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) - if uploadErr != nil { - log.Err("TieredStorage::uploadFile : %s upload failed [%v]", name, uploadErr) +func (q *lruQueue) worker() { + defer q.wg.Done() + for fileName := range q.uploadChan { + err := q.upload(fileName) + if err != nil { + log.Err("lruPolicy::worker : failed to upload file %s: %v", fileName, err) + } } - return uploadErr - } - -//ok so the worker is going to have to take a file and upload to cloud From ed56450b6ede075413cae7e21c71f00c40cbd6d5 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Wed, 1 Jul 2026 11:25:38 -0600 Subject: [PATCH 27/89] LRU capacity design fix --- component/tiered_storage/lru_policy.go | 39 ++++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 1a4059f28..d328b7dcb 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -2,6 +2,8 @@ package tiered_storage import ( "fmt" + "os" + "path/filepath" "sync" "time" @@ -67,7 +69,6 @@ func (q *lruQueue) StartPolicy() error { } func (q *lruQueue) StopPolicy() error { - close(q.doneChan) q.wg.Wait() return nil @@ -154,6 +155,9 @@ func (q *lruQueue) capacityChecker() { select { case <-time.After(2 * time.Minute): // eviction + + //check du , do stat file before, based on difference between DU and + //1. check if we need eviction curSize, err := common.GetUsage(q.cachePath) if err != nil { @@ -163,18 +167,27 @@ func (q *lruQueue) capacityChecker() { if curSize/q.maxCacheSize <= q.threshold { break } - for curSize/q.maxCacheSize > q.targetRatio { - if !q.eviction() { + + //targetRatio should always be less than thresholdRatio + + //find difference to evict down to 60% + difference := curSize - q.maxCacheSize*q.targetRatio + curEvictedSpace := 0 + for curEvictedSpace < int(difference) { + + nodeName, evicted := q.eviction() + if !evicted { break } - curSize, err = common.GetUsage(q.cachePath) + + localPath := filepath.Join(q.cachePath, nodeName) + + fileInfo, err := os.Stat(localPath) if err != nil { - log.Err( - "lruPolicy::capacityChecker : failed to get usage after eviction: %v", - err, - ) - continue + log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) + break } + curEvictedSpace += int(fileInfo.Size()) } case <-q.doneChan: @@ -183,13 +196,15 @@ func (q *lruQueue) capacityChecker() { } } -func (q *lruQueue) eviction() bool { +func (q *lruQueue) eviction() (string, bool) { q.mu.Lock() nodeToEvict := q.tail if nodeToEvict == nil { - return false + return "", false } + //ok we have to add in handle checkers/logic + //remove node from queue and map q.extractNode(nodeToEvict) q.nodeMap.Delete(nodeToEvict.name) @@ -197,7 +212,7 @@ func (q *lruQueue) eviction() bool { //send node to channel q.uploadChan <- nodeToEvict.name - return true + return nodeToEvict.name, true } func (q *lruQueue) worker() { From ea8b42afcf79fea5b2a9e53ebf114fc8c9da66b4 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Wed, 1 Jul 2026 14:38:16 -0600 Subject: [PATCH 28/89] Fixed deadlock bugs in channels --- component/tiered_storage/lru_policy.go | 57 +++++++++++++++----------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index d328b7dcb..e0b929633 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -81,14 +81,15 @@ func (q *lruQueue) Touch(name string) { func (q *lruQueue) Enqueue(name string) { //Maybe have a duplicate , that touches essentially + //lock earlier + q.mu.Lock() + defer q.mu.Unlock() + //create node newNode := &lruNode{name: name} val, found := q.nodeMap.LoadOrStore(name, newNode) node := val.(*lruNode) - q.mu.Lock() - defer q.mu.Unlock() - if found { // touch q.extractNode(node) @@ -104,14 +105,14 @@ func (q *lruQueue) Enqueue(name string) { func (q *lruQueue) Dequeue(name string) { log.Trace("lruPolicy::removeNode : %s", name) + q.mu.Lock() + defer q.mu.Unlock() + val, found := q.nodeMap.LoadAndDelete(name) if !found || val == nil { return } - q.mu.Lock() - defer q.mu.Unlock() - node := val.(*lruNode) q.extractNode(node) @@ -151,9 +152,12 @@ func (q *lruQueue) capacityChecker() { defer q.wg.Done() defer close(q.uploadChan) + ticker := time.NewTicker(2 * time.Minute) + defer ticker.Stop() + for { select { - case <-time.After(2 * time.Minute): + case <-ticker.C: // eviction //check du , do stat file before, based on difference between DU and @@ -174,20 +178,11 @@ func (q *lruQueue) capacityChecker() { difference := curSize - q.maxCacheSize*q.targetRatio curEvictedSpace := 0 for curEvictedSpace < int(difference) { - - nodeName, evicted := q.eviction() + nodeSize, evicted := q.eviction() if !evicted { break } - - localPath := filepath.Join(q.cachePath, nodeName) - - fileInfo, err := os.Stat(localPath) - if err != nil { - log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) - break - } - curEvictedSpace += int(fileInfo.Size()) + curEvictedSpace += int(nodeSize) } case <-q.doneChan: @@ -196,14 +191,25 @@ func (q *lruQueue) capacityChecker() { } } -func (q *lruQueue) eviction() (string, bool) { +func (q *lruQueue) eviction() (int64, bool) { q.mu.Lock() nodeToEvict := q.tail if nodeToEvict == nil { - return "", false + return 0, false } - //ok we have to add in handle checkers/logic + //ok we have to add in handle checkers/logic, only evict if no active handle, else touch to skip, + //Add in handle logic at the top to choose which node we want + + //Get the node size that we evict + localPath := filepath.Join(q.cachePath, nodeToEvict.name) + + fileInfo, err := os.Stat(localPath) + if err != nil { + log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) + return 0, false + } + nodeSize := fileInfo.Size() //remove node from queue and map q.extractNode(nodeToEvict) @@ -211,8 +217,13 @@ func (q *lruQueue) eviction() (string, bool) { q.mu.Unlock() //send node to channel - q.uploadChan <- nodeToEvict.name - return nodeToEvict.name, true + select { + case q.uploadChan <- nodeToEvict.name: + case <-q.doneChan: + return 0, false + } + + return nodeSize, true } func (q *lruQueue) worker() { From 3ea1864d5677a9bc1c5a4151cf06c7ad41ab384b Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 2 Jul 2026 11:16:39 -0600 Subject: [PATCH 29/89] Initial test file setup --- component/tiered_storage/lru_policy.go | 36 ++++++-- component/tiered_storage/lru_policy_test.go | 97 +++++++++++++++++++++ 2 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 component/tiered_storage/lru_policy_test.go diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index e0b929633..7e147bea4 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -39,8 +39,17 @@ type lruQueue struct { threshold float64 targetRatio float64 + + //Functions to wire later into tiered_storage package //upload function from tiered storage WIRE THIS LATER in tiered storage because we are using a function from there upload func(name string) error + + FileHasOpenFileHandle func(name string) bool + + //policy.isFileInUse = func(name string) bool { + // return c.fileLocks.Get(name).Count() > 0 + //} + } func (q *lruQueue) StartPolicy() error { @@ -81,7 +90,7 @@ func (q *lruQueue) Touch(name string) { func (q *lruQueue) Enqueue(name string) { //Maybe have a duplicate , that touches essentially - //lock earlier + //lock earlier q.mu.Lock() defer q.mu.Unlock() @@ -152,6 +161,7 @@ func (q *lruQueue) capacityChecker() { defer q.wg.Done() defer close(q.uploadChan) + ticker := time.NewTicker(2 * time.Minute) defer ticker.Stop() @@ -198,9 +208,25 @@ func (q *lruQueue) eviction() (int64, bool) { return 0, false } - //ok we have to add in handle checkers/logic, only evict if no active handle, else touch to skip, + //find the first applicable node + for nodeToEvict!= nil && q.FileHasOpenFileHandle(nodeToEvict.name){ + prevNode := nodeToEvict.prev + if q.tail == nil { + q.tail = nodeToEvict + } + q.setHead(nodeToEvict) + nodeToEvict = prevNode + } + if nodeToEvict == nil { + q.mu.Unlock() + return 0, false + } + + + //ok we have to add in handle checkers/logic, only evict if no active handle, else touch to skip, //Add in handle logic at the top to choose which node we want + //Get the node size that we evict localPath := filepath.Join(q.cachePath, nodeToEvict.name) @@ -217,12 +243,12 @@ func (q *lruQueue) eviction() (int64, bool) { q.mu.Unlock() //send node to channel - select { + select{ case q.uploadChan <- nodeToEvict.name: case <-q.doneChan: - return 0, false + return 0, false } - + return nodeSize, true } diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go new file mode 100644 index 000000000..e0d517bab --- /dev/null +++ b/component/tiered_storage/lru_policy_test.go @@ -0,0 +1,97 @@ +/* + Licensed under the MIT License . + + Copyright © 2023-2026 Seagate Technology LLC and/or its Affiliates + Copyright © 2020-2026 Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package tiered_storage + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type lruPolicyTestSuite struct { + suite.Suite + assert *assert.Assertions + policy *lruQueue +} + +var cache_path = filepath.Join(home_dir, "file_cache"+randomString(8)) + +func (suite *lruPolicyTestSuite) SetupTest() { + err := log.SetDefaultLogger("silent", common.LogConfig{Level: common.ELogLevel.LOG_DEBUG()}) + if err != nil { + panic(fmt.Sprintf("Unable to set silent logger as default: %v", err)) + } + suite.assert = assert.New(suite.T()) + + err = os.Mkdir(cache_path, fs.FileMode(0777)) + suite.assert.NoError(err) + + suite.setupTestHelper(cache_path, 1024, 0.8, 0.6, 2) +} + +func (suite *lruPolicyTestSuite) TeardownTest() { + err := suite.policy.StopPolicy() + suite.assert.NoError(err) + + err = os.RemoveAll(cache_path) + suite.assert.NoError(err) +} + +// setupTestHelper creates and starts an lruQueue for testing. +// cachePath: where cached files live +// maxCacheMB: max cache size in MB (converted to bytes internally) +// threshold: eviction triggers above this ratio (e.g. 0.8 = 80% full) +// targetRatio: evict down to this ratio (e.g. 0.6 = 60% full) +// numWorkers: number of upload worker goroutines +func (suite *lruPolicyTestSuite) setupTestHelper( + cachePath string, maxCacheMB float64, threshold float64, targetRatio float64, numWorkers int, +) { + suite.policy = &lruQueue{ + cachePath: cachePath, + maxCacheSize: maxCacheMB * 1024 * 1024, // convert MB to bytes + threshold: threshold, + targetRatio: targetRatio, + numWorkers: numWorkers, + + // Stub: tests don't upload to a real backend + upload: func(name string) error { + return nil + }, + // Stub: assume no file handles are open during tests + FileHasOpenFileHandle: func(name string) bool { + return false + }, + } + + err := suite.policy.StartPolicy() + suite.assert.NoError(err) +} From 4b15841a1a0666eae2bc1af900d54d1ae351b830 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 6 Jul 2026 14:17:07 -0600 Subject: [PATCH 30/89] Bug fixes and initial tests of lru eviction policy ready for pr draft --- component/tiered_storage/lru_policy.go | 40 +-- component/tiered_storage/lru_policy_test.go | 269 ++++++++++++++++++-- 2 files changed, 274 insertions(+), 35 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 7e147bea4..135f46672 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -39,6 +39,7 @@ type lruQueue struct { threshold float64 targetRatio float64 + tickerUnit time.Duration //Functions to wire later into tiered_storage package //upload function from tiered storage WIRE THIS LATER in tiered storage because we are using a function from there @@ -47,9 +48,11 @@ type lruQueue struct { FileHasOpenFileHandle func(name string) bool //policy.isFileInUse = func(name string) bool { - // return c.fileLocks.Get(name).Count() > 0 + // return c.fileLocks.Get(name).Count() > 0 //} + //we also need the filLock map to use + } func (q *lruQueue) StartPolicy() error { @@ -90,7 +93,7 @@ func (q *lruQueue) Touch(name string) { func (q *lruQueue) Enqueue(name string) { //Maybe have a duplicate , that touches essentially - //lock earlier + //lock earlier q.mu.Lock() defer q.mu.Unlock() @@ -131,7 +134,9 @@ func (q *lruQueue) setHead(node *lruNode) { // insert node at the head node.prev = nil node.next = q.head - q.head.prev = node + if q.head != nil { + q.head.prev = node + } q.head = node } @@ -161,8 +166,7 @@ func (q *lruQueue) capacityChecker() { defer q.wg.Done() defer close(q.uploadChan) - - ticker := time.NewTicker(2 * time.Minute) + ticker := time.NewTicker(q.tickerUnit) defer ticker.Stop() for { @@ -205,27 +209,24 @@ func (q *lruQueue) eviction() (int64, bool) { q.mu.Lock() nodeToEvict := q.tail if nodeToEvict == nil { + q.mu.Unlock() return 0, false } - //find the first applicable node - for nodeToEvict!= nil && q.FileHasOpenFileHandle(nodeToEvict.name){ - prevNode := nodeToEvict.prev - if q.tail == nil { - q.tail = nodeToEvict - } - q.setHead(nodeToEvict) + //find the first applicable node + for nodeToEvict != nil && q.FileHasOpenFileHandle(nodeToEvict.name) { + prevNode := nodeToEvict.prev + q.extractNode(nodeToEvict) + q.setHead(nodeToEvict) nodeToEvict = prevNode } + //means all files are in use, what if all files are in use so what do we evict then?????time?????? if nodeToEvict == nil { q.mu.Unlock() return 0, false } - - //ok we have to add in handle checkers/logic, only evict if no active handle, else touch to skip, - //Add in handle logic at the top to choose which node we want - + //right here is where we lock the file, where do we unlock it //Get the node size that we evict localPath := filepath.Join(q.cachePath, nodeToEvict.name) @@ -233,6 +234,7 @@ func (q *lruQueue) eviction() (int64, bool) { fileInfo, err := os.Stat(localPath) if err != nil { log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) + q.mu.Unlock() return 0, false } nodeSize := fileInfo.Size() @@ -243,12 +245,12 @@ func (q *lruQueue) eviction() (int64, bool) { q.mu.Unlock() //send node to channel - select{ + select { case q.uploadChan <- nodeToEvict.name: case <-q.doneChan: - return 0, false + return 0, false } - + return nodeSize, true } diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index e0d517bab..655ba296d 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -30,6 +30,9 @@ import ( "io/fs" "os" "path/filepath" + "sync" + "testing" + "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" @@ -55,23 +58,10 @@ func (suite *lruPolicyTestSuite) SetupTest() { err = os.Mkdir(cache_path, fs.FileMode(0777)) suite.assert.NoError(err) - suite.setupTestHelper(cache_path, 1024, 0.8, 0.6, 2) -} - -func (suite *lruPolicyTestSuite) TeardownTest() { - err := suite.policy.StopPolicy() - suite.assert.NoError(err) - - err = os.RemoveAll(cache_path) - suite.assert.NoError(err) + suite.setupTestHelper(cache_path, 1, 0.8, 0.6, 8) } // setupTestHelper creates and starts an lruQueue for testing. -// cachePath: where cached files live -// maxCacheMB: max cache size in MB (converted to bytes internally) -// threshold: eviction triggers above this ratio (e.g. 0.8 = 80% full) -// targetRatio: evict down to this ratio (e.g. 0.6 = 60% full) -// numWorkers: number of upload worker goroutines func (suite *lruPolicyTestSuite) setupTestHelper( cachePath string, maxCacheMB float64, threshold float64, targetRatio float64, numWorkers int, ) { @@ -81,12 +71,11 @@ func (suite *lruPolicyTestSuite) setupTestHelper( threshold: threshold, targetRatio: targetRatio, numWorkers: numWorkers, + tickerUnit: time.Millisecond, - // Stub: tests don't upload to a real backend upload: func(name string) error { return nil }, - // Stub: assume no file handles are open during tests FileHasOpenFileHandle: func(name string) bool { return false }, @@ -95,3 +84,251 @@ func (suite *lruPolicyTestSuite) setupTestHelper( err := suite.policy.StartPolicy() suite.assert.NoError(err) } + +func (suite *lruPolicyTestSuite) cleanupTest() { + err := suite.policy.StopPolicy() + suite.assert.NoError(err) + + err = os.RemoveAll(cache_path) + suite.assert.NoError(err) +} + +// Test +// 1. Touch +func (suite *lruPolicyTestSuite) TestTouch() { + defer suite.cleanupTest() + //put one file in + name := "file1" + fileName := filepath.Join(cache_path, name) + suite.policy.Touch(fileName) + suite.assert.Equal(fileName, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) + + //put another file in + name2 := "file2" + fileName2 := filepath.Join(cache_path, name2) + suite.policy.Touch(fileName2) + suite.assert.Equal(fileName2, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) + + //touch file1 back to top + suite.policy.Touch(fileName) + suite.assert.Equal(fileName, suite.policy.head.name) + suite.assert.Equal(fileName2, suite.policy.tail.name) +} + +// 2. enqueueItem +func (suite *lruPolicyTestSuite) TestEnqueue() { + defer suite.cleanupTest() + //put one file in + name := "file1" + fileName := filepath.Join(cache_path, name) + suite.policy.Enqueue(fileName) + suite.assert.Equal(fileName, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) + + //put another file in + name2 := "file2" + fileName2 := filepath.Join(cache_path, name2) + suite.policy.Enqueue(fileName2) + suite.assert.Equal(fileName2, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) + + //put another file in + name3 := "file3" + fileName3 := filepath.Join(cache_path, name3) + suite.policy.Enqueue(fileName3) + suite.assert.Equal(fileName3, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) +} + +// 3. Dequeue +func (suite *lruPolicyTestSuite) TestDequeue() { + defer suite.cleanupTest() + //put one file in + name := "file1" + fileName := filepath.Join(cache_path, name) + suite.policy.Enqueue(fileName) + suite.assert.Equal(fileName, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) + + //put another file in + name2 := "file2" + fileName2 := filepath.Join(cache_path, name2) + suite.policy.Enqueue(fileName2) + suite.assert.Equal(fileName2, suite.policy.head.name) + suite.assert.Equal(fileName, suite.policy.tail.name) + + //remove + suite.policy.Dequeue(fileName) + suite.assert.Equal(fileName2, suite.policy.head.name) + suite.assert.Equal(fileName2, suite.policy.tail.name) +} + +// 5. Capacity checker, two cases +func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { + defer suite.cleanupTest() + + var mu sync.Mutex + + //1. Define an arbitrary upload function to test the functionality of the channel + var uploaded []string + suite.policy.upload = func(name string) error { + mu.Lock() + uploaded = append(uploaded, name) + os.Remove(filepath.Join(cache_path, name)) + mu.Unlock() + return nil + } + + //2. Create files that exceed the 80% threshold, max set at 1MB + data := make([]byte, 250*1024) + err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file1") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file2") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file3"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file3") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file4"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file4") + + //file4 should be in upload channel + time.Sleep(10 * time.Millisecond) + + mu.Lock() + snapshot := make([]string, len(uploaded)) + copy(snapshot, uploaded) + mu.Unlock() + + // file1 and file2 are the LRU tail so they should be evicted to reach targetRatio (60%) + suite.assert.Contains(snapshot, "file1") + suite.assert.Contains(snapshot, "file2") + // file3 and file4 are the most recently used, so they should NOT be evicted + suite.assert.NotContains(snapshot, "file3") + suite.assert.NotContains(snapshot, "file4") +} + +// 6. Eviction, file with open handle, file with no open handle, +func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { + defer suite.cleanupTest() + var mu sync.Mutex + + //1. Define an arbitrary upload function to test the functionality of the channel + var uploaded []string + suite.policy.upload = func(name string) error { + mu.Lock() + uploaded = append(uploaded, name) + os.Remove(filepath.Join(cache_path, name)) + mu.Unlock() + return nil + } + + //give file1 an openFileHandle, should be touched to the top and skipped, so we expect file2 and 3 to be uploaded + suite.policy.FileHasOpenFileHandle = func(name string) bool { + if name == "file1" { + return true + } + return false + } + + //2. Create files that exceed the 80% threshold, max set at 1MB + data := make([]byte, 250*1024) + err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file1") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file2") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file3"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file3") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file4"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file4") + + //file4 should be in upload channel + time.Sleep(10 * time.Millisecond) + + mu.Lock() + snapshot := make([]string, len(uploaded)) + copy(snapshot, uploaded) + mu.Unlock() + + fmt.Print(snapshot) + fmt.Print(suite.policy.head.name) + fmt.Print(suite.policy.tail.name) + + //file1 should be head, file4 should be tail + suite.assert.Equal("file1", suite.policy.head.name) + suite.assert.Equal("file4", suite.policy.tail.name) + + // file1 and file2 are the LRU tail so they should be evicted to reach targetRatio (60%) + suite.assert.Contains(snapshot, "file2") + suite.assert.Contains(snapshot, "file3") + // file3 and file4 are the most recently used, so they should NOT be evicted + suite.assert.NotContains(snapshot, "file1") + suite.assert.NotContains(snapshot, "file4") + +} + +// 7. Test done channel function +func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionDoneChanDrain() { + //fill up upload chan + //call stop policy + //make sure all files in upload were indeed uploaded + + var mu sync.Mutex + + //1. Define an arbitrary upload function to test the functionality of the channel + var uploaded []string + suite.policy.upload = func(name string) error { + mu.Lock() + uploaded = append(uploaded, name) + os.Remove(filepath.Join(cache_path, name)) + mu.Unlock() + return nil + } + + suite.policy.uploadChan <- "file1" + suite.policy.uploadChan <- "file2" + suite.policy.uploadChan <- "file3" + suite.policy.uploadChan <- "file4" + + //Stop policy + err := suite.policy.StopPolicy() + suite.assert.NoError(err) + + time.Sleep(10 * time.Millisecond) + mu.Lock() + snapshot := make([]string, len(uploaded)) + copy(snapshot, uploaded) + mu.Unlock() + + suite.assert.Contains(snapshot, "file1") + suite.assert.Contains(snapshot, "file2") + suite.assert.Contains(snapshot, "file3") + suite.assert.Contains(snapshot, "file4") + + err = os.RemoveAll(cache_path) + suite.assert.NoError(err) +} + +func TestLRUPolicyTestSuite(t *testing.T) { + suite.Run(t, new(lruPolicyTestSuite)) +} From 5e54e560564efbbabb2302995d44b0f9d82d2af1 Mon Sep 17 00:00:00 2001 From: HoJacob Date: Mon, 6 Jul 2026 15:38:42 -0600 Subject: [PATCH 31/89] Feature/tiered storage/read in buffer (#959) * Implemented initial design of ReleaseFile and corresponding tests * Initial ReadInBuffer component * Added initial ReadInBuffer tests --------- Co-authored-by: Jacob Ho --- component/tiered_storage/tiered_storage.go | 17 +++++- .../tiered_storage/tiered_storage_test.go | 52 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 3cb1b31f9..96fd621bb 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -28,6 +28,7 @@ package tiered_storage import ( "context" "fmt" + "io" "os" "path/filepath" "sync" @@ -442,7 +443,21 @@ func (c *TieredStorage) isOverLocalLimit( } func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int, error) { - return 0, nil + f := options.Handle.GetFileObject() + if f == nil { + log.Err( + "TieredStorage::ReadInBuffer : error [couldn't find fd in handle] %s", + options.Handle.Path, + ) + return 0, syscall.EBADF + } + + n, err := f.ReadAt(options.Data, options.Offset) + // ReadAt gives an error if it reads fewer bytes than the byte array. We discard that error. + if n < len(options.Data) && err == io.EOF { + return n, nil + } + return n, err } func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 779877a4b..fadf0afab 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -391,6 +391,14 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { // Verify it was now downloaded to the local tiered storage cache suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + + // Handle should be dirty since it was not created in cloud storage + suite.assert.True(handle.Dirty()) + //As of now, the file would be cloudbacked and exist in map suite.tieredStorage.mu.Lock() node, exists := suite.tieredStorage.fileMap[path] @@ -410,6 +418,7 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { suite.assert.NoError(err) //tmpFile to hold cloud data || WARNING AI SLOP BELOW, I did not write below this + //It just checks if the data is preserved tmpFile, err := os.CreateTemp("", "cloud_verify") suite.assert.NoError(err) defer os.Remove(tmpFile.Name()) @@ -435,6 +444,49 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { } +func (suite *tieredStorageTestSuite) TestReadInBuffer() { + defer suite.cleanupTest() + // Setup + file := "file14" + + //put file in cloud abd write to it + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) + testData := "test data" + data := []byte(testData) + _, err := suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + err = suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //Must check that file by its data is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: file, RetrieveMetadata: true}) + suite.assert.NoError(err) + + handle, _ = suite.tieredStorage.OpenFile(internal.OpenFileOptions{Name: file, Mode: 0777}) + + output := make([]byte, 9) + length, err := suite.tieredStorage.ReadInBuffer( + &internal.ReadInBufferOptions{Handle: handle, Offset: 0, Data: output}, + ) + suite.assert.NoError(err) + suite.assert.Equal(data, output) + suite.assert.Equal(len(data), length) +} + +func (suite *tieredStorageTestSuite) TestReadInBufferErrorBadFd() { + defer suite.cleanupTest() + // Setup + file := "file15" + handle := handlemap.NewHandle(file) + length, err := suite.tieredStorage.ReadInBuffer(&internal.ReadInBufferOptions{Handle: handle}) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.EBADF, err) + suite.assert.Equal(0, length) +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 3f3a1b8e46505896cffa32cef697f32dd1aecc4b Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 13 Jul 2026 13:18:44 -0600 Subject: [PATCH 32/89] Handeled Dirty File State Scenario, ReleaseFile bugs, added Tests for dirty edge case --- component/tiered_storage/tiered_storage.go | 44 ++++---- .../tiered_storage/tiered_storage_test.go | 104 ++++++++++++++++++ 2 files changed, 127 insertions(+), 21 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 23ca608cf..bb149ae71 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -71,6 +71,7 @@ type FileNode struct { prev *FileNode next *FileNode cloudBacked bool + isDirty bool // Add more attributes as needed, e.g., last accessed time, etc. } @@ -223,6 +224,7 @@ func (c *TieredStorage) createFileUnlocked( name: options.Name, size: uint64(0), cloudBacked: false, + isDirty: true, } c.mu.Lock() c.fileMap[options.Name] = node @@ -535,6 +537,7 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro c.mu.Lock() if node, ok := c.fileMap[options.Handle.Path]; ok { node.size = uint64(newSize) + node.isDirty = true } c.mu.Unlock() } else { @@ -562,9 +565,21 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { flock.Lock() defer flock.Unlock() - //Dec Handle First + //Dec Handle Count First flock.Dec() + //close file associated with handle + if f := options.Handle.GetFileObject(); f != nil { + f.Close() + } + + //clean handle state + c.clearHandleDirty(options.Handle) + options.Handle.Cleanup() + + //remove from global handle map + handlemap.Delete(options.Handle.ID) + //Check if this is the last file handle handleCount := flock.Count() @@ -585,7 +600,7 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { if node.cloudBacked { //File was modified - if options.Handle.Dirty() { + if node.isDirty { //Upload err := c.uploadCachedFile(options.Handle.Path) if err != nil { @@ -594,30 +609,17 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { options.Handle.Path, err, ) - options.Handle.Cleanup() return err } - //Delete local file copy - localPath := filepath.Join(c.tmpPath, options.Handle.Path) - c.mu.Lock() - delete(c.fileMap, options.Handle.Path) - c.mu.Unlock() - //Clean Handle - options.Handle.Cleanup() - os.Remove(localPath) - } else { - //File was not modified - localPath := filepath.Join(c.tmpPath, options.Handle.Path) - c.mu.Lock() - delete(c.fileMap, options.Handle.Path) - c.mu.Unlock() - options.Handle.Cleanup() - os.Remove(localPath) - } + // Whether File was not modified or not, delete local file copy + localPath := filepath.Join(c.tmpPath, options.Handle.Path) + c.mu.Lock() + delete(c.fileMap, options.Handle.Path) + c.mu.Unlock() + os.Remove(localPath) } else { //local only then just close the file, update LRU add to queue, we will get to this later - options.Handle.Cleanup() } } return nil diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index fadf0afab..7eec87d3a 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -487,6 +487,110 @@ func (suite *tieredStorageTestSuite) TestReadInBufferErrorBadFd() { suite.assert.Equal(0, length) } +func (suite *tieredStorageTestSuite) TestWriteReadDirtyState() { + defer suite.cleanupTest() + path := "file16" + + //put file in cloud + handle, _ := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0777}) + err := suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + //open file through tiered storage, should succeed and return a handle with correct path + handle, openErr := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(openErr) + + // Verify it was now downloaded to the local tiered storage cache + in map + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + suite.tieredStorage.mu.Lock() + node, exists := suite.tieredStorage.fileMap[path] + suite.tieredStorage.mu.Unlock() + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(exists, "File should be tracked in the fileMap") + + //1. Write to handle + testData := "test data" + data := []byte(testData) + length, err := suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, + ) + suite.assert.NoError(err) + suite.assert.Equal(len(data), length) + + //check the handle is dirty + suite.assert.True(handle.Dirty()) + + //2. New Read Handle to same file + handle2, openErr := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{ + Name: path, + Flags: os.O_RDWR, + Mode: 0666, //random mode, since we didn't do the other stuff yet + }, + ) + suite.assert.NoError(openErr) + output := make([]byte, 9) + length, err = suite.tieredStorage.ReadInBuffer( + &internal.ReadInBufferOptions{Handle: handle2, Offset: 0, Data: output}, + ) + suite.assert.NoError(err) + suite.assert.Equal(data, output) + suite.assert.Equal(len(data), length) + + //3. Release The write handle, should still be in local with a dirty handle + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + _, err = os.Stat(filepath.Join(suite.cache_path, path)) + suite.assert.False(os.IsNotExist(err), "File should not be uploaded") + + //check still dirty + suite.assert.False(handle2.Dirty()) + suite.assert.False(handle.Dirty()) + + suite.tieredStorage.mu.Lock() + node, _ = suite.tieredStorage.fileMap[path] + suite.tieredStorage.mu.Unlock() + + suite.assert.True(node.isDirty, "File should be marked as dirty") + + //4. Release the read should upload to cloud + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle2}) + suite.assert.NoError(err) + _, err = os.Stat(filepath.Join(suite.cache_path, path)) + suite.assert.True(os.IsNotExist(err), "File should be deleted from cache after release") + + //5. Check data + //It just checks if the data is preserved + tmpFile, err := os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err := os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 74cce482263f70d307afb61f0fc19fbd15bee350 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 14 Jul 2026 10:14:42 -0600 Subject: [PATCH 33/89] Changed all instances of normal map to sync.map --- component/tiered_storage/tiered_storage.go | 116 +++++++++++++-------- 1 file changed, 75 insertions(+), 41 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index bb149ae71..38e8ca7e2 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -51,7 +51,9 @@ import ( // Common structure for Component type TieredStorage struct { internal.BaseComponent - fileMap map[string]*FileNode + //fileMap map[string]*FileNode + fileMap sync.Map + lruQueue *LRUQueue //use LockMap instead of mutex to allow parallel access to different files @@ -226,9 +228,11 @@ func (c *TieredStorage) createFileUnlocked( cloudBacked: false, isDirty: true, } - c.mu.Lock() - c.fileMap[options.Name] = node - c.mu.Unlock() + // c.mu.Lock() + // c.fileMap[options.Name] = node + // c.mu.Unlock() + + c.fileMap.Store(options.Name, node) //create handle handle := handlemap.NewHandle(options.Name) @@ -262,17 +266,22 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { defer flock.Unlock() //Check file map - c.mu.Lock() - node, exists := c.fileMap[options.Name] - c.mu.Unlock() + // c.mu.Lock() + // node, exists := c.fileMap[options.Name] + // c.mu.Unlock() + + val, exists := c.fileMap.Load(options.Name) + if exists { //local only + node := val.(*FileNode) if !node.cloudBacked { //delete locally localPath := filepath.Join(c.tmpPath, options.Name) - c.mu.Lock() - delete(c.fileMap, options.Name) - c.mu.Unlock() + // c.mu.Lock() + // delete(c.fileMap, options.Name) + // c.mu.Unlock() + c.fileMap.Delete(options.Name) os.Remove(localPath) //Both @@ -284,9 +293,11 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { } //delete locally localPath := filepath.Join(c.tmpPath, options.Name) - c.mu.Lock() - delete(c.fileMap, options.Name) - c.mu.Unlock() + // c.mu.Lock() + // delete(c.fileMap, options.Name) + // c.mu.Unlock() + c.fileMap.Delete(options.Name) + os.Remove(localPath) } @@ -314,9 +325,11 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H //Case 1: OpenFile with O_Create if options.Flags&os.O_CREATE != 0 { //Check if file first exists, then proceed - c.mu.Lock() - _, exists := c.fileMap[options.Name] - c.mu.Unlock() + // c.mu.Lock() + // _, exists := c.fileMap[options.Name] + // c.mu.Unlock() + + _, exists := c.fileMap.Load(options.Name) if !exists { handle, err := c.createFileUnlocked( internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, @@ -330,9 +343,10 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H } //1. Initial Check Map - c.mu.Lock() - _, exists := c.fileMap[options.Name] - c.mu.Unlock() + // c.mu.Lock() + // _, exists := c.fileMap[options.Name] + // c.mu.Unlock() + _, exists := c.fileMap.Load(options.Name) //if exists skip to opening file since it should already be in local cache if !exists { @@ -349,9 +363,12 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H size: uint64(info.Size()), cloudBacked: false, } - c.mu.Lock() - c.fileMap[options.Name] = node - c.mu.Unlock() + // c.mu.Lock() + // c.fileMap[options.Name] = node + // c.mu.Unlock() + + c.fileMap.Store(options.Name, node) + } else { //3. Check if File exists in Cloud info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) @@ -378,9 +395,12 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H if err != nil { return nil, err } - c.mu.Lock() - c.fileMap[options.Name] = localCopyNode - c.mu.Unlock() + // c.mu.Lock() + // c.fileMap[options.Name] = localCopyNode + // c.mu.Unlock() + + c.fileMap.Store(options.Name, localCopyNode) + } } @@ -460,11 +480,15 @@ func (c *TieredStorage) isOverLocalLimit( //find ExistingSize of file if exists existingSize := uint64(0) - c.mu.Lock() - if node, ok := c.fileMap[fileName]; ok { - existingSize = node.size + // c.mu.Lock() + // if node, ok := c.fileMap[fileName]; ok { + // existingSize = node.size + // } + // c.mu.Unlock() + + if val, ok := c.fileMap.Load(fileName); ok { + existingSize = val.(*FileNode).size } - c.mu.Unlock() addedFileSize := int64(newFileSize) - int64(existingSize) @@ -534,12 +558,18 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro if err == nil { c.setHandleDirty(options.Handle) //update file node size in file map - c.mu.Lock() - if node, ok := c.fileMap[options.Handle.Path]; ok { + // c.mu.Lock() + // if node, ok := c.fileMap[options.Handle.Path]; ok { + // node.size = uint64(newSize) + // node.isDirty = true + // } + // c.mu.Unlock() + if val, ok := c.fileMap.Load(options.Handle.Path); ok { + node := val.(*FileNode) node.size = uint64(newSize) node.isDirty = true } - c.mu.Unlock() + } else { log.Err( "TieredStorage::WriteFile : failed to write %s [%s]", @@ -586,11 +616,13 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { //it is the last handle if handleCount == 0 { //is file cloudbacked - c.mu.Lock() - node, exists := c.fileMap[options.Handle.Path] - c.mu.Unlock() + // c.mu.Lock() + // node, exists := c.fileMap[options.Handle.Path] + // c.mu.Unlock() - if !exists { + val, ok := c.fileMap.Load(options.Handle.Path) + node := val.(*FileNode) + if !ok { log.Err( "TieredStorage::ReleaseFile : internal error: file %s not found in map", options.Handle.Path, @@ -612,11 +644,13 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { return err } } - // Whether File was not modified or not, delete local file copy + //Whether File was modified or not, delete local file copy localPath := filepath.Join(c.tmpPath, options.Handle.Path) - c.mu.Lock() - delete(c.fileMap, options.Handle.Path) - c.mu.Unlock() + // c.mu.Lock() + // delete(c.fileMap, options.Handle.Path) + // c.mu.Unlock() + c.fileMap.Delete(options.Handle.Path) + os.Remove(localPath) } else { //local only then just close the file, update LRU add to queue, we will get to this later @@ -724,7 +758,7 @@ func (c *TieredStorage) StatFs() (*common.Statfs_t, bool, error) { // << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> func NewTieredStorageComponent() internal.Component { comp := &TieredStorage{ - fileMap: make(map[string]*FileNode), + //fileMap: make(map[string]*FileNode), lruQueue: &LRUQueue{}, fileLocks: common.NewLockMap(), } From adef71ec041861fb5b14080a20793c11ab5a27ae Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 14 Jul 2026 10:23:29 -0600 Subject: [PATCH 34/89] Changed all instances of normal map to sync.map in TEST FILE NOW --- .../tiered_storage/tiered_storage_test.go | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 7eec87d3a..fa29cbf37 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -349,11 +349,15 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudNoDirtyFile() { suite.assert.FileExists(filepath.Join(suite.cache_path, path)) //As of now, the file would be cloudbacked and exist in map - suite.tieredStorage.mu.Lock() - node, exists := suite.tieredStorage.fileMap[path] - suite.tieredStorage.mu.Unlock() + // suite.tieredStorage.mu.Lock() + // node, exists := suite.tieredStorage.fileMap[path] + // suite.tieredStorage.mu.Unlock() + + val, ok := suite.tieredStorage.fileMap.Load(path) + node := val.(*FileNode) + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") - suite.assert.True(exists, "File should be tracked in the fileMap") + suite.assert.True(ok, "File should be tracked in the fileMap") //File should be "cloudBacked" and not dirty so on release the file should be deleted from local and the handle clean err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) @@ -400,9 +404,13 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { suite.assert.True(handle.Dirty()) //As of now, the file would be cloudbacked and exist in map - suite.tieredStorage.mu.Lock() - node, exists := suite.tieredStorage.fileMap[path] - suite.tieredStorage.mu.Unlock() + // suite.tieredStorage.mu.Lock() + // node, exists := suite.tieredStorage.fileMap[path] + // suite.tieredStorage.mu.Unlock() + + val, exists := suite.tieredStorage.fileMap.Load(path) + node := val.(*FileNode) + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") suite.assert.True(exists, "File should be tracked in the fileMap") @@ -508,9 +516,13 @@ func (suite *tieredStorageTestSuite) TestWriteReadDirtyState() { // Verify it was now downloaded to the local tiered storage cache + in map suite.assert.FileExists(filepath.Join(suite.cache_path, path)) - suite.tieredStorage.mu.Lock() - node, exists := suite.tieredStorage.fileMap[path] - suite.tieredStorage.mu.Unlock() + // suite.tieredStorage.mu.Lock() + // node, exists := suite.tieredStorage.fileMap[path] + // suite.tieredStorage.mu.Unlock() + + val, exists := suite.tieredStorage.fileMap.Load(path) + node := val.(*FileNode) + suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") suite.assert.True(exists, "File should be tracked in the fileMap") @@ -553,9 +565,12 @@ func (suite *tieredStorageTestSuite) TestWriteReadDirtyState() { suite.assert.False(handle2.Dirty()) suite.assert.False(handle.Dirty()) - suite.tieredStorage.mu.Lock() - node, _ = suite.tieredStorage.fileMap[path] - suite.tieredStorage.mu.Unlock() + // suite.tieredStorage.mu.Lock() + // node, _ = suite.tieredStorage.fileMap[path] + // suite.tieredStorage.mu.Unlock() + + val, _ = suite.tieredStorage.fileMap.Load(path) + node = val.(*FileNode) suite.assert.True(node.isDirty, "File should be marked as dirty") From e9dcb70d4e4b9b50e5695ead48c588de28f8a7a4 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 17 Jul 2026 13:07:58 -0600 Subject: [PATCH 35/89] LRU Concurrency Design Implementation --- component/tiered_storage/lru_policy.go | 195 ++++++++++++++++++------- 1 file changed, 141 insertions(+), 54 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 135f46672..ae3eac9ca 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "time" "github.com/Seagate/cloudfuse/common" @@ -24,39 +25,54 @@ type lruQueue struct { nodeMap sync.Map - wg sync.WaitGroup - numWorkers int + wg sync.WaitGroup // tracks capacityChecker only + workerWg sync.WaitGroup // tracks worker goroutines + numWorkers int + activeWorkers int32 head *lruNode tail *lruNode - uploadChan chan string - doneChan chan struct{} + uploadChan chan string + doneChan chan struct{} + hallPassChan chan bool - cachePath string - maxCacheSize float64 + cachePath string + maxCacheSize float64 + totalUploadedSize int64 threshold float64 targetRatio float64 tickerUnit time.Duration + fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) + //Functions to wire later into tiered_storage package //upload function from tiered storage WIRE THIS LATER in tiered storage because we are using a function from there - upload func(name string) error + uploadFn func(name string) error + //this function is just used for testing FileHasOpenFileHandle func(name string) bool //policy.isFileInUse = func(name string) bool { // return c.fileLocks.Get(name).Count() > 0 //} - //we also need the filLock map to use + //we also wire a cleanup function to delete from FileMap and Local + cleanupFn func(name string) error + + // //delete locally + // localPath := filepath.Join(c.tmpPath, options.Name) + // c.mu.Lock() + // delete(c.fileMap, options.Name) + // c.mu.Unlock() + // os.Remove(localPath) } func (q *lruQueue) StartPolicy() error { - if q.upload == nil { + if q.uploadFn == nil { return fmt.Errorf("lruQueue: upload function not set") } if q.numWorkers <= 0 { @@ -66,23 +82,25 @@ func (q *lruQueue) StartPolicy() error { q.head = nil q.tail = nil //channels - q.uploadChan = make(chan string, 1000) q.doneChan = make(chan struct{}) + q.hallPassChan = make(chan bool, 1) + q.hallPassChan <- true + //timer //go routines q.wg.Add(1) go q.capacityChecker() - q.wg.Add(q.numWorkers) - for i := 0; i < q.numWorkers; i++ { - go q.worker() - } return nil } func (q *lruQueue) StopPolicy() error { close(q.doneChan) + // Wait for capacityChecker to exit — its deferred close(uploadChan) fires here, + // signalling workers that no more jobs are coming. q.wg.Wait() + // Now wait for workers to drain whatever remains in uploadChan. + q.workerWg.Wait() return nil } @@ -173,32 +191,63 @@ func (q *lruQueue) capacityChecker() { select { case <-ticker.C: // eviction - - //check du , do stat file before, based on difference between DU and - - //1. check if we need eviction - curSize, err := common.GetUsage(q.cachePath) - if err != nil { - log.Err("lruPolicy::capacityChecker : failed to get usage: %v", err) - continue - } - if curSize/q.maxCacheSize <= q.threshold { - break - } - - //targetRatio should always be less than thresholdRatio - - //find difference to evict down to 60% - difference := curSize - q.maxCacheSize*q.targetRatio - curEvictedSpace := 0 - for curEvictedSpace < int(difference) { - nodeSize, evicted := q.eviction() - if !evicted { + select { + case <-q.hallPassChan: + + //1. Check if we need eviction + curSize, err := common.GetUsage(q.cachePath) + if err != nil { + log.Err("lruPolicy::capacityChecker : failed to get usage: %v", err) + q.hallPassChan <- true + continue + } + if curSize/q.maxCacheSize <= q.threshold { + q.hallPassChan <- true break } - curEvictedSpace += int(nodeSize) - } + //targetRatio should always be less than thresholdRatio + + //2. Find difference to evict down to target ratio + difference := curSize - q.maxCacheSize*q.targetRatio + curEvictedSpace := 0 + actualEvictedSpace := 0 + atomic.StoreInt64(&q.totalUploadedSize, 0) + + //3. LRU Eviction to match difference + for actualEvictedSpace < int(difference) { + + //initialize channel + q.uploadChan = make(chan string, 1000) + + //start workers here + q.workerWg.Add(q.numWorkers) + for i := 0; i < q.numWorkers; i++ { + go q.worker() + } + + //populate upload channel for workers + for curEvictedSpace < int(difference) { + nodeSize, evicted := q.eviction() + if !evicted { + break + } + curEvictedSpace += int(nodeSize) + } + + //close upload chan and wait for workers to process all remaining jobs + close(q.uploadChan) + q.workerWg.Wait() + + //update the actual evicted space here + actualEvictedSpace = int(q.totalUploadedSize) + } + //give hall pass back when actual evicted space is satisfied + q.hallPassChan <- true + + case <-q.doneChan: + return + } case <-q.doneChan: return } @@ -213,40 +262,51 @@ func (q *lruQueue) eviction() (int64, bool) { return 0, false } - //find the first applicable node - for nodeToEvict != nil && q.FileHasOpenFileHandle(nodeToEvict.name) { + //1. loop through and find the first applicable node + for nodeToEvict != nil { prevNode := nodeToEvict.prev + + flock := q.fileLocks.Get(nodeToEvict.name) + flock.RLock() + handleCount := flock.Count() + flock.RUnlock() + + if handleCount == 0 { + break + } + + //node has open handles touch node q.extractNode(nodeToEvict) q.setHead(nodeToEvict) nodeToEvict = prevNode } - //means all files are in use, what if all files are in use so what do we evict then?????time?????? + + //all files are in use if nodeToEvict == nil { q.mu.Unlock() return 0, false } + name := nodeToEvict.name - //right here is where we lock the file, where do we unlock it + //2. Remove file from queue so not accidentally chosen again + q.extractNode(nodeToEvict) + q.nodeMap.Delete(name) + + q.mu.Unlock() - //Get the node size that we evict - localPath := filepath.Join(q.cachePath, nodeToEvict.name) + //3. Get the node size that we evict + localPath := filepath.Join(q.cachePath, name) fileInfo, err := os.Stat(localPath) if err != nil { log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) - q.mu.Unlock() return 0, false } nodeSize := fileInfo.Size() - //remove node from queue and map - q.extractNode(nodeToEvict) - q.nodeMap.Delete(nodeToEvict.name) - q.mu.Unlock() - - //send node to channel + //4. Send node to channel to be uploaded by workers select { - case q.uploadChan <- nodeToEvict.name: + case q.uploadChan <- name: case <-q.doneChan: return 0, false } @@ -255,11 +315,38 @@ func (q *lruQueue) eviction() (int64, bool) { } func (q *lruQueue) worker() { - defer q.wg.Done() + defer q.workerWg.Done() for fileName := range q.uploadChan { - err := q.upload(fileName) + + //1. Get handle count and file size + flock := q.fileLocks.Get(fileName) + flock.Lock() + handleCount := flock.Count() + + localPath := filepath.Join(q.cachePath, fileName) + fileInfo, err := os.Stat(localPath) if err != nil { - log.Err("lruPolicy::worker : failed to upload file %s: %v", fileName, err) + log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) + } + + fileSize := fileInfo.Size() + + //2. Check if file eligible to upload + if handleCount == 0 { + err := q.uploadandCleanFn(fileName) + flock.Unlock() + if err != nil { + log.Err("lruPolicy::worker : failed to upload file %s: %v", fileName, err) + //if upload fails we have to put the file back to the queue to retry later + q.Touch(fileName) + + } else { + atomic.AddInt64(&q.totalUploadedSize, fileSize) + } + //file is in use skip upload touch file back to top of queue + } else { + flock.Unlock() + q.Touch(fileName) } } } From 13c608c9b7bc982433e579d63a966b6998ff119a Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 20 Jul 2026 13:58:24 -0600 Subject: [PATCH 36/89] Fixed infinite loop logic and StopPolicy, old tests now work also test for Shutdown --- component/tiered_storage/lru_policy.go | 52 +++++++++-------- component/tiered_storage/lru_policy_test.go | 65 +++++++++++++-------- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index ae3eac9ca..3de859f53 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -50,17 +50,7 @@ type lruQueue struct { //Functions to wire later into tiered_storage package //upload function from tiered storage WIRE THIS LATER in tiered storage because we are using a function from there - uploadFn func(name string) error - - //this function is just used for testing - FileHasOpenFileHandle func(name string) bool - - //policy.isFileInUse = func(name string) bool { - // return c.fileLocks.Get(name).Count() > 0 - //} - - //we also wire a cleanup function to delete from FileMap and Local - cleanupFn func(name string) error + uploadandCleanFn func(name string) error // //delete locally // localPath := filepath.Join(c.tmpPath, options.Name) @@ -72,7 +62,7 @@ type lruQueue struct { } func (q *lruQueue) StartPolicy() error { - if q.uploadFn == nil { + if q.uploadandCleanFn == nil { return fmt.Errorf("lruQueue: upload function not set") } if q.numWorkers <= 0 { @@ -86,7 +76,6 @@ func (q *lruQueue) StartPolicy() error { q.hallPassChan = make(chan bool, 1) q.hallPassChan <- true - //timer //go routines q.wg.Add(1) go q.capacityChecker() @@ -115,21 +104,23 @@ func (q *lruQueue) Enqueue(name string) { q.mu.Lock() defer q.mu.Unlock() - //create node - newNode := &lruNode{name: name} - val, found := q.nodeMap.LoadOrStore(name, newNode) - node := val.(*lruNode) - + //search for new node first + val, found := q.nodeMap.Load(name) if found { - // touch + // Node already exists, no need to create a new one, just touch + node := val.(*lruNode) q.extractNode(node) + q.setHead(node) } else { - // brand new node — update tail if list was empty + // Node does not exist need to create a new one and put to top + newNode := &lruNode{name: name} + // if list is empty, set tail pointer to newNode if q.tail == nil { - q.tail = node + q.tail = newNode } + q.nodeMap.Store(name, newNode) + q.setHead(newNode) } - q.setHead(node) } func (q *lruQueue) Dequeue(name string) { @@ -182,7 +173,7 @@ func (q *lruQueue) extractNode(node *lruNode) { func (q *lruQueue) capacityChecker() { defer q.wg.Done() - defer close(q.uploadChan) + //defer close(q.uploadChan) ticker := time.NewTicker(q.tickerUnit) defer ticker.Stop() @@ -215,6 +206,9 @@ func (q *lruQueue) capacityChecker() { //3. LRU Eviction to match difference for actualEvictedSpace < int(difference) { + // Start nomination from what's already been confirmed uploaded, + // so we only nominate enough new files to cover the remaining gap. + curEvictedSpace = actualEvictedSpace //initialize channel q.uploadChan = make(chan string, 1000) @@ -240,6 +234,16 @@ func (q *lruQueue) capacityChecker() { //update the actual evicted space here actualEvictedSpace = int(q.totalUploadedSize) + + //if done is closed we need to check shutdown to prevent infinite loop + select { + case <-q.doneChan: + return + default: + } + + //we can also have a case here if no files were evicted then we can break on this tick if we so choose + } //give hall pass back when actual evicted space is satisfied q.hallPassChan <- true @@ -247,6 +251,8 @@ func (q *lruQueue) capacityChecker() { case <-q.doneChan: return + //default return? + default: } case <-q.doneChan: return diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index 655ba296d..c69d8630b 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -72,13 +72,11 @@ func (suite *lruPolicyTestSuite) setupTestHelper( targetRatio: targetRatio, numWorkers: numWorkers, tickerUnit: time.Millisecond, + fileLocks: common.NewLockMap(), // required by eviction() and worker() - upload: func(name string) error { + uploadandCleanFn: func(name string) error { return nil }, - FileHasOpenFileHandle: func(name string) bool { - return false - }, } err := suite.policy.StartPolicy() @@ -173,7 +171,7 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { //1. Define an arbitrary upload function to test the functionality of the channel var uploaded []string - suite.policy.upload = func(name string) error { + suite.policy.uploadandCleanFn = func(name string) error { mu.Lock() uploaded = append(uploaded, name) os.Remove(filepath.Join(cache_path, name)) @@ -225,7 +223,7 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { //1. Define an arbitrary upload function to test the functionality of the channel var uploaded []string - suite.policy.upload = func(name string) error { + suite.policy.uploadandCleanFn = func(name string) error { mu.Lock() uploaded = append(uploaded, name) os.Remove(filepath.Join(cache_path, name)) @@ -233,20 +231,18 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { return nil } - //give file1 an openFileHandle, should be touched to the top and skipped, so we expect file2 and 3 to be uploaded - suite.policy.FileHasOpenFileHandle = func(name string) bool { - if name == "file1" { - return true - } - return false - } - //2. Create files that exceed the 80% threshold, max set at 1MB data := make([]byte, 250*1024) err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) suite.assert.NoError(err) suite.policy.Enqueue("file1") + //open a file handle for file1 so it should get skipped and touched to the top, file1, file4, file3, file2 + flock := suite.policy.fileLocks.Get("file1") + flock.Lock() + flock.Inc() + flock.Unlock() + data = make([]byte, 250*1024) err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) suite.assert.NoError(err) @@ -262,7 +258,6 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { suite.assert.NoError(err) suite.policy.Enqueue("file4") - //file4 should be in upload channel time.Sleep(10 * time.Millisecond) mu.Lock() @@ -288,7 +283,7 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { } // 7. Test done channel function -func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionDoneChanDrain() { +func (suite *lruPolicyTestSuite) TestStopPolicyMidUpload() { //fill up upload chan //call stop policy //make sure all files in upload were indeed uploaded @@ -297,33 +292,53 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionDoneChanDrain() { //1. Define an arbitrary upload function to test the functionality of the channel var uploaded []string - suite.policy.upload = func(name string) error { + suite.policy.uploadandCleanFn = func(name string) error { mu.Lock() + //make upload super slow + time.Sleep(200 * time.Millisecond) uploaded = append(uploaded, name) os.Remove(filepath.Join(cache_path, name)) mu.Unlock() return nil } - suite.policy.uploadChan <- "file1" - suite.policy.uploadChan <- "file2" - suite.policy.uploadChan <- "file3" - suite.policy.uploadChan <- "file4" + data := make([]byte, 250*1024) + err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file1") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file2") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file3"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file3") + + data = make([]byte, 250*1024) + err = os.WriteFile(filepath.Join(cache_path, "file4"), data, 0644) + suite.assert.NoError(err) + suite.policy.Enqueue("file4") + + time.Sleep(5 * time.Millisecond) //Stop policy - err := suite.policy.StopPolicy() + err = suite.policy.StopPolicy() suite.assert.NoError(err) - time.Sleep(10 * time.Millisecond) mu.Lock() snapshot := make([]string, len(uploaded)) copy(snapshot, uploaded) mu.Unlock() + fmt.Print(snapshot) + suite.assert.Contains(snapshot, "file1") suite.assert.Contains(snapshot, "file2") - suite.assert.Contains(snapshot, "file3") - suite.assert.Contains(snapshot, "file4") + suite.assert.NotContains(snapshot, "file3") + suite.assert.NotContains(snapshot, "file4") err = os.RemoveAll(cache_path) suite.assert.NoError(err) From 8ac166bea156a7dd550932ce52a02d1e19cc96de Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 20 Jul 2026 14:34:33 -0600 Subject: [PATCH 37/89] Initial wiring of LRU Policy --- component/tiered_storage/tiered_storage.go | 51 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 38e8ca7e2..bd192b462 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -33,6 +33,7 @@ import ( "path/filepath" "sync" "syscall" + "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -54,7 +55,7 @@ type TieredStorage struct { //fileMap map[string]*FileNode fileMap sync.Map - lruQueue *LRUQueue + policy *lruQueue //use LockMap instead of mutex to allow parallel access to different files fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) @@ -120,6 +121,14 @@ func (c *TieredStorage) Start(ctx context.Context) error { // TieredStorage : start code goes here + //Start the policy + if c.policy != nil { + if err := c.policy.StartPolicy(); err != nil { + log.Err("TieredStorage::Start : failed to start LRU policy [%v]", err) + return err + } + } + return nil } @@ -127,6 +136,10 @@ func (c *TieredStorage) Start(ctx context.Context) error { func (c *TieredStorage) Stop() error { log.Trace("TieredStorage::Stop : Stopping component %s", c.Name()) + if c.policy != nil { + return c.policy.StopPolicy() + } + return nil } @@ -156,6 +169,20 @@ func (c *TieredStorage) Configure(_ bool) error { return fmt.Errorf("TieredStorage: failed to create tmp path: %w", err) } + //figure out the maxCache size stuff here, there is just a bunch of configure stuff that we need to figure out + + //Wire in the LRU Policy + c.policy = &lruQueue{ + cachePath: c.tmpPath, + maxCacheSize: c.maxCacheSize, + fileLocks: c.fileLocks, + threshold: 0.8, + targetRatio: 0.6, + numWorkers: 8, + tickerUnit: time.Millisecond, + uploadandCleanFn: c.uploadandCleanFile, + } + return nil } @@ -653,7 +680,10 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { os.Remove(localPath) } else { - //local only then just close the file, update LRU add to queue, we will get to this later + // update LRU add to queue because cleaning up the file should be handled once the file is uploaded in LRU policy logic + if c.policy != nil { + c.policy.Enqueue(options.Handle.Path) + } } } return nil @@ -684,6 +714,21 @@ func (c *TieredStorage) uploadCachedFile(name string) error { return uploadErr } +func (c *TieredStorage) uploadandCleanFile(name string) error { + err := c.uploadCachedFile(name) + if err != nil { + return err + } + localPath := filepath.Join(c.tmpPath, name) + c.fileMap.Delete(name) + err = os.Remove(localPath) + if err != nil { + log.Err("TieredStorage::uploadandCleanFile : %s remove failed [%v]", name, err) + return err + } + return nil +} + func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { return nil } @@ -758,8 +803,6 @@ func (c *TieredStorage) StatFs() (*common.Statfs_t, bool, error) { // << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> func NewTieredStorageComponent() internal.Component { comp := &TieredStorage{ - //fileMap: make(map[string]*FileNode), - lruQueue: &LRUQueue{}, fileLocks: common.NewLockMap(), } comp.SetName(compName) From d6c9a305b6ecc604558b0545563ed0cb20daf00c Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 21 Jul 2026 15:43:27 -0600 Subject: [PATCH 38/89] Wired LRU Policy with Tiered Storage, Implemented Tests against functionality --- component/tiered_storage/lru_policy.go | 9 +- component/tiered_storage/lru_policy_test.go | 34 +++ component/tiered_storage/tiered_storage.go | 5 +- .../tiered_storage/tiered_storage_test.go | 202 +++++++++++++++++- 4 files changed, 242 insertions(+), 8 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 3de859f53..1a9a3bf16 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -88,8 +88,6 @@ func (q *lruQueue) StopPolicy() error { // Wait for capacityChecker to exit — its deferred close(uploadChan) fires here, // signalling workers that no more jobs are coming. q.wg.Wait() - // Now wait for workers to drain whatever remains in uploadChan. - q.workerWg.Wait() return nil } @@ -203,9 +201,10 @@ func (q *lruQueue) capacityChecker() { curEvictedSpace := 0 actualEvictedSpace := 0 atomic.StoreInt64(&q.totalUploadedSize, 0) + evicFail := false //3. LRU Eviction to match difference - for actualEvictedSpace < int(difference) { + for actualEvictedSpace < int(difference) && !evicFail { // Start nomination from what's already been confirmed uploaded, // so we only nominate enough new files to cover the remaining gap. curEvictedSpace = actualEvictedSpace @@ -223,6 +222,7 @@ func (q *lruQueue) capacityChecker() { for curEvictedSpace < int(difference) { nodeSize, evicted := q.eviction() if !evicted { + evicFail = true break } curEvictedSpace += int(nodeSize) @@ -343,9 +343,8 @@ func (q *lruQueue) worker() { flock.Unlock() if err != nil { log.Err("lruPolicy::worker : failed to upload file %s: %v", fileName, err) - //if upload fails we have to put the file back to the queue to retry later + //if upload fails we have to put the file back to the queue and map to retry later q.Touch(fileName) - } else { atomic.AddInt64(&q.totalUploadedSize, fileSize) } diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index c69d8630b..d154f6ce5 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -200,6 +200,16 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { suite.assert.NoError(err) suite.policy.Enqueue("file4") + _, ex1 := suite.policy.nodeMap.Load("file1") + _, ex2 := suite.policy.nodeMap.Load("file2") + _, ex3 := suite.policy.nodeMap.Load("file3") + _, ex4 := suite.policy.nodeMap.Load("file4") + + suite.assert.True(ex1) + suite.assert.True(ex2) + suite.assert.True(ex3) + suite.assert.True(ex4) + //file4 should be in upload channel time.Sleep(10 * time.Millisecond) @@ -214,6 +224,30 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { // file3 and file4 are the most recently used, so they should NOT be evicted suite.assert.NotContains(snapshot, "file3") suite.assert.NotContains(snapshot, "file4") + + fmt.Println("=== nodeMap contents ===") + suite.policy.nodeMap.Range(func(key, value interface{}) bool { + lruNode := value.(*lruNode) + fmt.Printf(" key=%q, name=%q, next=%v, prev=%v\n", + key, + lruNode.name, + lruNode.next, + lruNode.prev, + ) + return true // continue iteration + }) + fmt.Println("=== end nodeMap ===") + + _, ex1 = suite.policy.nodeMap.Load("file1") + _, ex2 = suite.policy.nodeMap.Load("file2") + _, ex3 = suite.policy.nodeMap.Load("file3") + _, ex4 = suite.policy.nodeMap.Load("file4") + + suite.assert.False(ex1) + suite.assert.False(ex2) + suite.assert.True(ex3) + suite.assert.True(ex4) + } // 6. Eviction, file with open handle, file with no open handle, diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index bd192b462..dc82a4fd4 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -89,7 +89,8 @@ type LRUQueue struct { // Structure defining your config parameters type TieredStorageOptions struct { // e.g. var1 uint32 `config:"var1"` - TmpPath string `config:"path" yaml:"path,omitempty"` + TmpPath string `config:"path" yaml:"path,omitempty"` + MaxSizeMB float64 `config:"max-size-mb" yaml:"max-size-mb,omitempty"` } const ( @@ -170,7 +171,7 @@ func (c *TieredStorage) Configure(_ bool) error { } //figure out the maxCache size stuff here, there is just a bunch of configure stuff that we need to figure out - + c.maxCacheSize = conf.MaxSizeMB * 1024 * 1024 //Wire in the LRU Policy c.policy = &lruQueue{ cachePath: c.tmpPath, diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index fa29cbf37..73adce416 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -9,6 +9,7 @@ import ( "strings" "syscall" "testing" + "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -71,7 +72,7 @@ func (suite *tieredStorageTestSuite) SetupTest() { suite.cache_path = filepath.Join(home_dir, "file_cache"+rand) suite.fake_storage_path = filepath.Join(home_dir, "fake_storage"+rand) defaultConfig := fmt.Sprintf( - "tiered_storage:\n path: %s\n offload-io: true\n\nloopbackfs:\n path: %s", + "tiered_storage:\n path: %s\n max-size-mb: 1.0\n offload-io: true\n\nloopbackfs:\n path: %s", suite.cache_path, suite.fake_storage_path, ) @@ -606,6 +607,205 @@ func (suite *tieredStorageTestSuite) TestWriteReadDirtyState() { ) } +func (suite *tieredStorageTestSuite) TestReleaseLocalToLRUQueue() { + //Ok this next test is to essentially go through an iteration of LRU, + + //1. Initialize a local only file + defer suite.cleanupTest() + path := "file17" + handle, err := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path, handle.Path) + suite.assert.True(handle.Dirty()) + // File should exist in cache + suite.assert.FileExists(filepath.Join(suite.cache_path, path)) + + val, exists := suite.tieredStorage.fileMap.Load(path) + node := val.(*FileNode) + + suite.assert.False(node.cloudBacked, "File should not be marked as cloud-backed") + suite.assert.True(exists, "File should be tracked in the fileMap") + + // 2. Release this local file + //File is local only so it shouldn't be deleted from local knowledge + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + // 3. Check if its in the LRU Queue + suite.assert.Equal(path, suite.tieredStorage.policy.head.name) + suite.assert.Equal(path, suite.tieredStorage.policy.tail.name) + +} + +func (suite *tieredStorageTestSuite) TestReleaseToTriggerEviction() { + // Ok this next test is to essentially go through an iteration of LRU, + // 1. Initialize many local only file + //2. Create files that exceed the 80% threshold, max set at 1MB + data := make([]byte, 250*1024) + path1 := "file18" + handle, err := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path1, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path1, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + path2 := "file19" + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path2, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path2, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + path3 := "file20" + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path3, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path3, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + path4 := "file21" + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path4, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path4, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + // 3. Check if all in the LRU Queue initially + suite.assert.Equal(path4, suite.tieredStorage.policy.head.name) + suite.assert.Equal(path3, suite.tieredStorage.policy.head.next.name) + suite.assert.Equal(path2, suite.tieredStorage.policy.head.next.next.name) + suite.assert.Equal(path1, suite.tieredStorage.policy.tail.name) + + _, exists1 := suite.tieredStorage.policy.nodeMap.Load(path1) + _, exists2 := suite.tieredStorage.policy.nodeMap.Load(path2) + _, exists3 := suite.tieredStorage.policy.nodeMap.Load(path3) + _, exists4 := suite.tieredStorage.policy.nodeMap.Load(path4) + + suite.assert.True(exists1) + suite.assert.True(exists2) + suite.assert.True(exists3) + suite.assert.True(exists4) + + //4. Sleep to wait for eviction to kick in + time.Sleep(100 * time.Millisecond) + + // 4. Some should then be released to the cloud essentially, the ones we wrote data to + //And the local files should be gone (uploaded and cleaned up), not in either map + + // 4a. Check state of NodeMap + _, exists1 = suite.tieredStorage.policy.nodeMap.Load(path1) + _, exists2 = suite.tieredStorage.policy.nodeMap.Load(path2) + _, exists3 = suite.tieredStorage.policy.nodeMap.Load(path3) + _, exists4 = suite.tieredStorage.policy.nodeMap.Load(path4) + + suite.assert.False(exists1) + suite.assert.False(exists2) + suite.assert.True(exists3) + suite.assert.True(exists4) + + //4b. Check state of fileMap + _, exists1 = suite.tieredStorage.fileMap.Load(path1) + _, exists2 = suite.tieredStorage.fileMap.Load(path2) + _, exists3 = suite.tieredStorage.fileMap.Load(path3) + _, exists4 = suite.tieredStorage.fileMap.Load(path4) + + suite.assert.False(exists1) + suite.assert.False(exists2) + suite.assert.True(exists3) + suite.assert.True(exists4) + + // 4c.Check files for files 1 and 2 no longer exist local + suite.assert.NoFileExists(filepath.Join(suite.cache_path, path1)) + suite.assert.NoFileExists(filepath.Join(suite.cache_path, path2)) + + //5. We have to check that the files exist in the cloud + //Must check that file is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: path1, RetrieveMetadata: true}) + suite.assert.NoError(err) + + //Must check that file is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: path2, RetrieveMetadata: true}) + suite.assert.NoError(err) + + //Validate the data matches what we have + //It just checks if the data is preserved + tmpFile, err := os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path1, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err := os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) + + //It just checks if the data is preserved + tmpFile, err = os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path2, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err = os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) + +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 7a58ff60718d5124e38bf937eb1a28e62fea952b Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Fri, 31 Jul 2026 14:46:07 -0600 Subject: [PATCH 39/89] Fixed DeleteFile and added in logic for edge case in LRU Policy --- component/tiered_storage/lru_policy.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 1a9a3bf16..eface82a2 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -332,7 +332,13 @@ func (q *lruQueue) worker() { localPath := filepath.Join(q.cachePath, fileName) fileInfo, err := os.Stat(localPath) if err != nil { - log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) + log.Warn( + "lruPolicy::worker : file %s no longer exists or stat failed, skipping: %v", + fileName, + err, + ) + flock.Unlock() + continue } fileSize := fileInfo.Size() @@ -341,10 +347,18 @@ func (q *lruQueue) worker() { if handleCount == 0 { err := q.uploadandCleanFn(fileName) flock.Unlock() + //handle when file doesn't exist during upload we do not requeue otherwise we do enqueue if err != nil { - log.Err("lruPolicy::worker : failed to upload file %s: %v", fileName, err) - //if upload fails we have to put the file back to the queue and map to retry later - q.Touch(fileName) + if os.IsNotExist(err) { + log.Warn( + "lruPolicy::worker : file %s was deleted during upload, skipping", + fileName, + ) + } else { + log.Err("lruPolicy::worker : failed to upload file %s: %v", fileName, err) + //if upload fails we have to put the file back to the queue and map to retry later + q.Touch(fileName) + } } else { atomic.AddInt64(&q.totalUploadedSize, fileSize) } From 776d9b36ab0d7478bd7159f587201b1686cc342e Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Mon, 3 Aug 2026 14:51:17 -0600 Subject: [PATCH 40/89] Tests for DeleteFile, still working on LRU Test --- .../tiered_storage/tiered_storage_test.go | 224 ++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 73adce416..4c1f333da 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -806,6 +806,230 @@ func (suite *tieredStorageTestSuite) TestReleaseToTriggerEviction() { } +// ok we gonna do file in local, cloud, file doesnt exist +func (suite *tieredStorageTestSuite) TestDeleteFileCloud() { + defer suite.cleanupTest() + // Setup + file := "file22" + + //put file in cloud abd write to it + handle, err := suite.tieredStorage.CreateFile( + internal.CreateFileOptions{Name: file, Mode: 0777}, + ) + suite.assert.NoError(err) + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + err = suite.tieredStorage.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.NoError(err) + + // Path should not be in file cache + suite.assert.NoFileExists(filepath.Join(suite.cache_path, file)) + + //file should not exist in cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: file, RetrieveMetadata: true}) + suite.assert.Error(err) + +} + +func (suite *tieredStorageTestSuite) TestDeleteFileLocal() { + defer suite.cleanupTest() + // Setup + file := "file23" + + //create local file + _, err := suite.tieredStorage.CreateFile( + internal.CreateFileOptions{Name: file, Mode: 0777}, + ) + suite.assert.NoError(err) + + err = suite.tieredStorage.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.NoError(err) + + // Path should not be in file cache + suite.assert.NoFileExists(filepath.Join(suite.cache_path, file)) + +} + +func (suite *tieredStorageTestSuite) TestDeleteFileNotExists() { + defer suite.cleanupTest() + // Setup + file := "file24" + + err := suite.tieredStorage.DeleteFile(internal.DeleteFileOptions{Name: file}) + suite.assert.Error(err) + suite.assert.EqualValues(syscall.ENOENT, err) +} + +// Ok lets set up a situation where the worker gets a bunch of files, we delete the first file right before upload, then check +// that the rest of the files do get uploaded to some cloud the error is successfully logged, we can make it super easy +func (suite *tieredStorageTestSuite) TestDeleteFileWorkerResponse() { + //1. Initialize many local only file + //1a. Create files that exceed the 80% threshold, max set at 1MB + data := make([]byte, 250*1024) + path1 := "file18" + handle, err := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path1, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path1, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + path2 := "file19" + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path2, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path2, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + path3 := "file20" + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path3, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path3, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + path4 := "file21" + handle, err = suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path4, Flags: os.O_CREATE, Mode: 0777}, + ) + suite.assert.NoError(err) + suite.assert.Equal(path4, handle.Path) + + suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.assert.True(handle.Dirty()) + + err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) + suite.assert.NoError(err) + + // 3. Check if all in the LRU Queue initially + suite.assert.Equal(path4, suite.tieredStorage.policy.head.name) + suite.assert.Equal(path3, suite.tieredStorage.policy.head.next.name) + suite.assert.Equal(path2, suite.tieredStorage.policy.head.next.next.name) + suite.assert.Equal(path1, suite.tieredStorage.policy.tail.name) + + _, exists1 := suite.tieredStorage.policy.nodeMap.Load(path1) + _, exists2 := suite.tieredStorage.policy.nodeMap.Load(path2) + _, exists3 := suite.tieredStorage.policy.nodeMap.Load(path3) + _, exists4 := suite.tieredStorage.policy.nodeMap.Load(path4) + + suite.assert.True(exists1) + suite.assert.True(exists2) + suite.assert.True(exists3) + suite.assert.True(exists4) + + //4. Sleep to wait for eviction to kick in + time.Sleep(100 * time.Millisecond) + + // 4. Some should then be released to the cloud essentially, the ones we wrote data to + //And the local files should be gone (uploaded and cleaned up), not in either map + + // 4a. Check state of NodeMap + _, exists1 = suite.tieredStorage.policy.nodeMap.Load(path1) + _, exists2 = suite.tieredStorage.policy.nodeMap.Load(path2) + _, exists3 = suite.tieredStorage.policy.nodeMap.Load(path3) + _, exists4 = suite.tieredStorage.policy.nodeMap.Load(path4) + + suite.assert.False(exists1) + suite.assert.False(exists2) + suite.assert.True(exists3) + suite.assert.True(exists4) + + //4b. Check state of fileMap + _, exists1 = suite.tieredStorage.fileMap.Load(path1) + _, exists2 = suite.tieredStorage.fileMap.Load(path2) + _, exists3 = suite.tieredStorage.fileMap.Load(path3) + _, exists4 = suite.tieredStorage.fileMap.Load(path4) + + suite.assert.False(exists1) + suite.assert.False(exists2) + suite.assert.True(exists3) + suite.assert.True(exists4) + + // 4c.Check files for files 1 and 2 no longer exist local + suite.assert.NoFileExists(filepath.Join(suite.cache_path, path1)) + suite.assert.NoFileExists(filepath.Join(suite.cache_path, path2)) + + //5. We have to check that the files exist in the cloud + //Must check that file is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: path1, RetrieveMetadata: true}) + suite.assert.NoError(err) + + //Must check that file is actually in the cloud + _, err = suite.tieredStorage.NextComponent().GetAttr( + internal.GetAttrOptions{Name: path2, RetrieveMetadata: true}) + suite.assert.NoError(err) + + //Validate the data matches what we have + //It just checks if the data is preserved + tmpFile, err := os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path1, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err := os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) + + //It just checks if the data is preserved + tmpFile, err = os.CreateTemp("", "cloud_verify") + suite.assert.NoError(err) + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // 2. Copy from the cloud (loopback) to the temporary file + err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ + Name: path2, + Offset: 0, + Count: 0, // 0 usually means the whole file + File: tmpFile, + }) + suite.assert.NoError(err) + + // 3. Read the data back from the temp file and verify + dataFromCloud, err = os.ReadFile(tmpFile.Name()) + suite.assert.NoError(err) + suite.assert.Equal( + data, + dataFromCloud, + "The cloud version should match the modified local version", + ) + +} + func TestTieredStorageTestSuite(t *testing.T) { suite.Run(t, new(tieredStorageTestSuite)) } From 5a99492096db15aaafcbcefab72112883d7c9a23 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 4 Aug 2026 11:48:12 -0600 Subject: [PATCH 41/89] Potential Fix for deadlock in Enqueue and Dequeue, manually unlocked in Release and Delete --- component/tiered_storage/tiered_storage.go | 68 +++++++++------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index dc82a4fd4..d8dc10e9b 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -288,56 +288,41 @@ func (c *TieredStorage) CreateFile( } func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { + log.Trace("TieredStorage::DeleteFile : name=%s", options.Name) //Lock the file first flock := c.fileLocks.Get(options.Name) flock.Lock() defer flock.Unlock() - //Check file map - // c.mu.Lock() - // node, exists := c.fileMap[options.Name] - // c.mu.Unlock() - val, exists := c.fileMap.Load(options.Name) - + //Potential local or local + cloud state if exists { - //local only node := val.(*FileNode) - if !node.cloudBacked { - //delete locally - localPath := filepath.Join(c.tmpPath, options.Name) - // c.mu.Lock() - // delete(c.fileMap, options.Name) - // c.mu.Unlock() - c.fileMap.Delete(options.Name) - os.Remove(localPath) + localPath := filepath.Join(c.tmpPath, options.Name) - //Both - } else { - //delete from cloud + //Local and Cloud State + if node.cloudBacked { + //Both local and cloud state + //delete from cloud first err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) if err != nil { return err } - //delete locally - localPath := filepath.Join(c.tmpPath, options.Name) - // c.mu.Lock() - // delete(c.fileMap, options.Name) - // c.mu.Unlock() - c.fileMap.Delete(options.Name) - - os.Remove(localPath) } + //Local only State + //remove from LRU if it is in there already and delete local file + c.fileMap.Delete(options.Name) + c.policy.Dequeue(options.Name) + os.Remove(localPath) + //Cloud only state } else { - //check cloud, else return an error + //delete from cloud err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) if err != nil { - return syscall.ENOENT + return err } - return err } - return nil } @@ -621,7 +606,9 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released flock := c.fileLocks.Get(options.Handle.Path) flock.Lock() - defer flock.Unlock() + + //Ok we have to manually unlock the file now instead + //defer flock.Unlock() //Dec Handle Count First flock.Dec() @@ -644,20 +631,16 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { //it is the last handle if handleCount == 0 { //is file cloudbacked - // c.mu.Lock() - // node, exists := c.fileMap[options.Handle.Path] - // c.mu.Unlock() - val, ok := c.fileMap.Load(options.Handle.Path) - node := val.(*FileNode) if !ok { log.Err( "TieredStorage::ReleaseFile : internal error: file %s not found in map", options.Handle.Path, ) + flock.Unlock() return syscall.EBADF } - + node := val.(*FileNode) if node.cloudBacked { //File was modified if node.isDirty { @@ -669,23 +652,26 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { options.Handle.Path, err, ) + flock.Unlock() return err } } //Whether File was modified or not, delete local file copy localPath := filepath.Join(c.tmpPath, options.Handle.Path) - // c.mu.Lock() - // delete(c.fileMap, options.Handle.Path) - // c.mu.Unlock() c.fileMap.Delete(options.Handle.Path) - os.Remove(localPath) + flock.Unlock() } else { // update LRU add to queue because cleaning up the file should be handled once the file is uploaded in LRU policy logic + //Unlock the file first so we don't hold two locks at the same time + flock.Unlock() if c.policy != nil { c.policy.Enqueue(options.Handle.Path) } } + } else { + //if not the last handle + flock.Unlock() } return nil } From 37507f98329ddce8149dff5934f812a37bd9a8a1 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 4 Aug 2026 11:50:01 -0600 Subject: [PATCH 42/89] Something might be wrong with my vsCode that commit might not have worked --- component/tiered_storage/tiered_storage.go | 34 ++++------------------ 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index d8dc10e9b..a31fbfac6 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -292,7 +292,9 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { //Lock the file first flock := c.fileLocks.Get(options.Name) flock.Lock() - defer flock.Unlock() + + //Unlock manually so we only hold one lock at a time + //defer flock.Unlock() val, exists := c.fileMap.Load(options.Name) //Potential local or local + cloud state @@ -306,18 +308,21 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { //delete from cloud first err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) if err != nil { + flock.Unlock() return err } } //Local only State //remove from LRU if it is in there already and delete local file c.fileMap.Delete(options.Name) + flock.Unlock() c.policy.Dequeue(options.Name) os.Remove(localPath) //Cloud only state } else { //delete from cloud + flock.Unlock() err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) if err != nil { return err @@ -338,10 +343,6 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H //Case 1: OpenFile with O_Create if options.Flags&os.O_CREATE != 0 { //Check if file first exists, then proceed - // c.mu.Lock() - // _, exists := c.fileMap[options.Name] - // c.mu.Unlock() - _, exists := c.fileMap.Load(options.Name) if !exists { handle, err := c.createFileUnlocked( @@ -356,9 +357,6 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H } //1. Initial Check Map - // c.mu.Lock() - // _, exists := c.fileMap[options.Name] - // c.mu.Unlock() _, exists := c.fileMap.Load(options.Name) //if exists skip to opening file since it should already be in local cache @@ -376,10 +374,6 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H size: uint64(info.Size()), cloudBacked: false, } - // c.mu.Lock() - // c.fileMap[options.Name] = node - // c.mu.Unlock() - c.fileMap.Store(options.Name, node) } else { @@ -408,10 +402,6 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H if err != nil { return nil, err } - // c.mu.Lock() - // c.fileMap[options.Name] = localCopyNode - // c.mu.Unlock() - c.fileMap.Store(options.Name, localCopyNode) } @@ -493,12 +483,6 @@ func (c *TieredStorage) isOverLocalLimit( //find ExistingSize of file if exists existingSize := uint64(0) - // c.mu.Lock() - // if node, ok := c.fileMap[fileName]; ok { - // existingSize = node.size - // } - // c.mu.Unlock() - if val, ok := c.fileMap.Load(fileName); ok { existingSize = val.(*FileNode).size } @@ -571,12 +555,6 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro if err == nil { c.setHandleDirty(options.Handle) //update file node size in file map - // c.mu.Lock() - // if node, ok := c.fileMap[options.Handle.Path]; ok { - // node.size = uint64(newSize) - // node.isDirty = true - // } - // c.mu.Unlock() if val, ok := c.fileMap.Load(options.Handle.Path); ok { node := val.(*FileNode) node.size = uint64(newSize) From ed6d5edb8b71b8d1ab34372077025e69e9480f97 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Tue, 4 Aug 2026 13:49:30 -0600 Subject: [PATCH 43/89] Initial implementation of Sync and Flush, they are identical --- component/tiered_storage/tiered_storage.go | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index a31fbfac6..49e6a05ee 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -573,10 +573,39 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro } func (c *TieredStorage) SyncFile(options internal.SyncFileOptions) error { - return nil + log.Trace( + "TieredStorage::SyncFile : handle=%d, path=%s", + options.Handle.ID, + options.Handle.Path, + ) + return c.FlushFile(internal.FlushFileOptions{Handle: options.Handle}) } func (c *TieredStorage) FlushFile(options internal.FlushFileOptions) error { + //Ok so we just need to flush locally, which means just write it to the disc + log.Trace( + "TieredStorage::FlushFile : handle=%d, path=%s", + options.Handle.ID, + options.Handle.Path, + ) + + //1. Only need to flush dirty files + if !options.Handle.Dirty() { + return nil + } + //2. Check if there is local file object form handle + f := options.Handle.GetFileObject() + if f == nil { + log.Err("TieredStorage::FlushFile : %s no file object in handle", options.Handle.Path) + return syscall.EBADF + } + //3. Sync to Disk + err := f.Sync() + if err != nil { + log.Err("TieredStorage::FlushFile : %s sync failed [%v]", options.Handle.Path, err) + return syscall.EIO + } + return nil } From 880ab0ed3d28efcd6f9e839a49e66c2dd200ddd2 Mon Sep 17 00:00:00 2001 From: Jacob Ho Date: Thu, 6 Aug 2026 16:44:27 -0600 Subject: [PATCH 44/89] WOW FINAL COMMIT MESSAGE SO COOL, added initial design of RenameFile, and finished test for FLushFile, THANK YOU GUYS SO MUCH, I hope everything goes well with this component and it gets pushed, Jacob --- component/tiered_storage/tiered_storage.go | 91 ++++++++++ .../tiered_storage/tiered_storage_test.go | 170 ++---------------- 2 files changed, 104 insertions(+), 157 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 49e6a05ee..9fe0eb053 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -724,9 +724,100 @@ func (c *TieredStorage) uploadandCleanFile(name string) error { } func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { + //Ok we are going to follow DeleteFile, we have to rename this File in the various states that its in + //So First we lock in alphabetical order + log.Trace("TieredStorage::RenameFile : src=%s, dst=%s", options.Src, options.Dst) + + sflock := c.fileLocks.Get(options.Src) + dflock := c.fileLocks.Get(options.Dst) + + if options.Src < options.Dst { + sflock.Lock() + dflock.Lock() + } else { + dflock.Lock() + sflock.Lock() + } + defer sflock.Unlock() + defer dflock.Unlock() + + //Ok now we have to consider all the states + //Rename File that is Local Only + //sync map in both local and in the LRU, also need to rename the node in the queue + //Check that it exists + val, exists := c.fileMap.Load(options.Src) + //Potential local or local + cloud state + if exists { + node := val.(*FileNode) + // //Local and Cloud State + if node.cloudBacked { + //just rename from the cloud + err := c.NextComponent().RenameFile(options) + if err != nil { + return err + } + } + //Local only State, this will happen anyways if it exists local + //Rename + err := os.Rename( + filepath.Join(c.tmpPath, options.Src), + filepath.Join(c.tmpPath, options.Dst), + ) + if err != nil { + return err + } + c.fileMap.Delete(options.Src) + node.name = options.Dst + c.fileMap.Store(options.Dst, node) + + //Check if it is in the LRU first + _, inLRU := c.policy.nodeMap.Load(options.Src) + if inLRU { + c.policy.Dequeue(options.Src) + c.policy.Enqueue(options.Dst) + } + //Change the handle and the lock counts + c.renameOpenHandles(options.Src, options.Dst, sflock, dflock) + + //Cloud only state + } else { + err := c.NextComponent().RenameFile(options) + if err != nil { + return err + } + } return nil } +// flock must be locked for both files +func (c *TieredStorage) renameOpenHandles( + srcName, dstName string, + sflock, dflock *common.LockMapItem, +) { + // update open handles + if sflock.Count() > 0 { + // update any open handles to the file with its new name + handlemap.GetHandles().Range(func(key, value any) bool { + handle := value.(*handlemap.Handle) + handle.Lock() + if handle.Path == srcName { + handle.Path = dstName + } + handle.Unlock() + return true + }) + // copy the number of open handles to the new name + for sflock.Count() > 0 { + sflock.Dec() + dflock.Inc() + } + for sflock.DirtyCount() > 0 { + sflock.DecDirty() + dflock.IncDirty() + } + } +} + func (c *TieredStorage) SyncDir(options internal.SyncDirOptions) error { return nil } diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 4c1f333da..05972148d 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -862,172 +862,28 @@ func (suite *tieredStorageTestSuite) TestDeleteFileNotExists() { suite.assert.EqualValues(syscall.ENOENT, err) } -// Ok lets set up a situation where the worker gets a bunch of files, we delete the first file right before upload, then check -// that the rest of the files do get uploaded to some cloud the error is successfully logged, we can make it super easy -func (suite *tieredStorageTestSuite) TestDeleteFileWorkerResponse() { - //1. Initialize many local only file - //1a. Create files that exceed the 80% threshold, max set at 1MB - data := make([]byte, 250*1024) - path1 := "file18" - handle, err := suite.tieredStorage.OpenFile( - internal.OpenFileOptions{Name: path1, Flags: os.O_CREATE, Mode: 0777}, - ) - suite.assert.NoError(err) - suite.assert.Equal(path1, handle.Path) - - suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) - suite.assert.True(handle.Dirty()) - - err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) - suite.assert.NoError(err) - - path2 := "file19" - handle, err = suite.tieredStorage.OpenFile( - internal.OpenFileOptions{Name: path2, Flags: os.O_CREATE, Mode: 0777}, - ) - suite.assert.NoError(err) - suite.assert.Equal(path2, handle.Path) - - suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) - suite.assert.True(handle.Dirty()) - - err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) - suite.assert.NoError(err) +func (suite *tieredStorageTestSuite) TestFlushFile() { + defer suite.cleanupTest() + file := "file25" + handle, _ := suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) - path3 := "file20" - handle, err = suite.tieredStorage.OpenFile( - internal.OpenFileOptions{Name: path3, Flags: os.O_CREATE, Mode: 0777}, + testData := "test data" + data := []byte(testData) + _, err := suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: data}, ) suite.assert.NoError(err) - suite.assert.Equal(path3, handle.Path) - - suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) suite.assert.True(handle.Dirty()) - err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) - suite.assert.NoError(err) - - path4 := "file21" - handle, err = suite.tieredStorage.OpenFile( - internal.OpenFileOptions{Name: path4, Flags: os.O_CREATE, Mode: 0777}, - ) + err = suite.tieredStorage.FlushFile(internal.FlushFileOptions{Handle: handle}) suite.assert.NoError(err) - suite.assert.Equal(path4, handle.Path) - suite.tieredStorage.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + //Verify Data is still on the disk + d, _ := os.ReadFile(filepath.Join(suite.cache_path, file)) + suite.assert.Equal(data, d) + //Check that handle is still dirty suite.assert.True(handle.Dirty()) - err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}) - suite.assert.NoError(err) - - // 3. Check if all in the LRU Queue initially - suite.assert.Equal(path4, suite.tieredStorage.policy.head.name) - suite.assert.Equal(path3, suite.tieredStorage.policy.head.next.name) - suite.assert.Equal(path2, suite.tieredStorage.policy.head.next.next.name) - suite.assert.Equal(path1, suite.tieredStorage.policy.tail.name) - - _, exists1 := suite.tieredStorage.policy.nodeMap.Load(path1) - _, exists2 := suite.tieredStorage.policy.nodeMap.Load(path2) - _, exists3 := suite.tieredStorage.policy.nodeMap.Load(path3) - _, exists4 := suite.tieredStorage.policy.nodeMap.Load(path4) - - suite.assert.True(exists1) - suite.assert.True(exists2) - suite.assert.True(exists3) - suite.assert.True(exists4) - - //4. Sleep to wait for eviction to kick in - time.Sleep(100 * time.Millisecond) - - // 4. Some should then be released to the cloud essentially, the ones we wrote data to - //And the local files should be gone (uploaded and cleaned up), not in either map - - // 4a. Check state of NodeMap - _, exists1 = suite.tieredStorage.policy.nodeMap.Load(path1) - _, exists2 = suite.tieredStorage.policy.nodeMap.Load(path2) - _, exists3 = suite.tieredStorage.policy.nodeMap.Load(path3) - _, exists4 = suite.tieredStorage.policy.nodeMap.Load(path4) - - suite.assert.False(exists1) - suite.assert.False(exists2) - suite.assert.True(exists3) - suite.assert.True(exists4) - - //4b. Check state of fileMap - _, exists1 = suite.tieredStorage.fileMap.Load(path1) - _, exists2 = suite.tieredStorage.fileMap.Load(path2) - _, exists3 = suite.tieredStorage.fileMap.Load(path3) - _, exists4 = suite.tieredStorage.fileMap.Load(path4) - - suite.assert.False(exists1) - suite.assert.False(exists2) - suite.assert.True(exists3) - suite.assert.True(exists4) - - // 4c.Check files for files 1 and 2 no longer exist local - suite.assert.NoFileExists(filepath.Join(suite.cache_path, path1)) - suite.assert.NoFileExists(filepath.Join(suite.cache_path, path2)) - - //5. We have to check that the files exist in the cloud - //Must check that file is actually in the cloud - _, err = suite.tieredStorage.NextComponent().GetAttr( - internal.GetAttrOptions{Name: path1, RetrieveMetadata: true}) - suite.assert.NoError(err) - - //Must check that file is actually in the cloud - _, err = suite.tieredStorage.NextComponent().GetAttr( - internal.GetAttrOptions{Name: path2, RetrieveMetadata: true}) - suite.assert.NoError(err) - - //Validate the data matches what we have - //It just checks if the data is preserved - tmpFile, err := os.CreateTemp("", "cloud_verify") - suite.assert.NoError(err) - defer os.Remove(tmpFile.Name()) - defer tmpFile.Close() - - // 2. Copy from the cloud (loopback) to the temporary file - err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ - Name: path1, - Offset: 0, - Count: 0, // 0 usually means the whole file - File: tmpFile, - }) - suite.assert.NoError(err) - - // 3. Read the data back from the temp file and verify - dataFromCloud, err := os.ReadFile(tmpFile.Name()) - suite.assert.NoError(err) - suite.assert.Equal( - data, - dataFromCloud, - "The cloud version should match the modified local version", - ) - - //It just checks if the data is preserved - tmpFile, err = os.CreateTemp("", "cloud_verify") - suite.assert.NoError(err) - defer os.Remove(tmpFile.Name()) - defer tmpFile.Close() - - // 2. Copy from the cloud (loopback) to the temporary file - err = suite.loopback.CopyToFile(internal.CopyToFileOptions{ - Name: path2, - Offset: 0, - Count: 0, // 0 usually means the whole file - File: tmpFile, - }) - suite.assert.NoError(err) - - // 3. Read the data back from the temp file and verify - dataFromCloud, err = os.ReadFile(tmpFile.Name()) - suite.assert.NoError(err) - suite.assert.Equal( - data, - dataFromCloud, - "The cloud version should match the modified local version", - ) - } func TestTieredStorageTestSuite(t *testing.T) { From db88ac7804d021d78439b0cffbbd6c58522d4836 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Wed, 26 Aug 2026 17:46:39 -0600 Subject: [PATCH 45/89] Add copyright notice --- component/tiered_storage/lru_policy.go | 24 +++++++++++++++++++ .../tiered_storage/tiered_storage_test.go | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index eface82a2..c5b90c278 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -1,3 +1,27 @@ +/* + Licensed under the MIT License . + + Copyright © 2026 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + package tiered_storage import ( diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 05972148d..25e381cc2 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -1,3 +1,27 @@ +/* + Licensed under the MIT License . + + Copyright © 2026 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + package tiered_storage import ( From c741105b4540a5966a9a813e96a9d78a47bb841d Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 27 Aug 2026 13:59:56 -0600 Subject: [PATCH 46/89] Make handleCount atomic --- common/lock_map.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/common/lock_map.go b/common/lock_map.go index bfbada53d..f734d5c3f 100644 --- a/common/lock_map.go +++ b/common/lock_map.go @@ -33,7 +33,7 @@ import ( // Lock item for each file type LockMapItem struct { - handleCount uint32 + handleCount atomic.Uint32 dirtyCount atomic.Uint32 mtx sync.RWMutex downloadTime time.Time @@ -54,7 +54,7 @@ func NewLockMap() *LockMap { // Get the lock item based on file name, if item does not exists create it func (l *LockMap) Get(name string) *LockMapItem { - lockIntf, _ := l.locks.LoadOrStore(name, &LockMapItem{handleCount: 0}) + lockIntf, _ := l.locks.LoadOrStore(name, &LockMapItem{}) item := lockIntf.(*LockMapItem) return item } @@ -75,6 +75,10 @@ func (l *LockMapItem) Unlock() { l.mtx.Unlock() } +func (l *LockMapItem) TryLock() bool { + return l.mtx.TryLock() +} + func (l *LockMapItem) RLock() { l.mtx.RLock() } @@ -86,17 +90,25 @@ func (l *LockMapItem) RUnlock() { // Increment the handle count func (l *LockMapItem) Inc() { - l.handleCount++ + l.handleCount.Add(1) } // Decrement the handle count func (l *LockMapItem) Dec() { - l.handleCount-- + for { + current := l.handleCount.Load() + if current == 0 { + return + } + if l.handleCount.CompareAndSwap(current, current-1) { + return + } + } } // Get the current handle count func (l *LockMapItem) Count() uint32 { - return l.handleCount + return l.handleCount.Load() } // Increment dirty-handle count. From 1b1631c5f14a9b0a3416239d9c279f02c7c4fe01 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Thu, 27 Aug 2026 17:08:36 -0600 Subject: [PATCH 47/89] Opus 5: TS 5-10 --- component/tiered_storage/lru_policy.go | 617 +++++++++++------- component/tiered_storage/lru_policy_test.go | 7 +- component/tiered_storage/tiered_storage.go | 473 ++++++++------ .../tiered_storage/tiered_storage_test.go | 10 +- 4 files changed, 672 insertions(+), 435 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index c5b90c278..2a797a03b 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -36,53 +36,99 @@ import ( "github.com/Seagate/cloudfuse/common/log" ) +const ( + // evictionPassLimit caps how many nominate-and-wait rounds one eviction + // cycle may run, so a cache that cannot be drained does not spin. + evictionPassLimit = 8 + + // maxRetryBackoff bounds the delay applied to an object whose upload keeps + // failing. + maxRetryBackoff = 5 * time.Minute + + // uploadQueueDepthPerWorker sizes the job channel. It only needs enough + // slack that the dispatcher is not serialised against the workers. + uploadQueueDepthPerWorker = 16 +) + +// nodeState records whether the queue or an eviction worker owns a node. +type nodeState uint8 + +const ( + // nodeQueued: linked into the list and eligible for eviction. + nodeQueued nodeState = iota + // nodeEvicting: unlinked and owned by a worker. The node deliberately stays + // in nodeMap so a rename or delete can still find it and cancel the + // eviction. Dropping it from the map here instead would leave a renamed + // object in no queue at all, never to be uploaded or evicted again. + nodeEvicting + // nodeCancelled: the object was deleted or renamed while a worker owned it. + // The worker must skip the upload and drop the node. + nodeCancelled +) + type lruNode struct { prev *lruNode next *lruNode name string + + state nodeState + // consecutive failed uploads, and the time before which this object should + // not be nominated again + failures uint32 + retryAfter time.Time } -//upload 50 files and then check +// uploadJob is one nominated object. The waitgroup and counters belong to the +// eviction pass that nominated it, so a pass waits only for its own batch. +type uploadJob struct { + name string + wg *sync.WaitGroup + freed *atomic.Int64 + evicted *atomic.Int32 +} +// lruQueue decides which local-only objects move to cloud storage. +// +// Locking: callers may hold a file lock and then take mu. The reverse is +// forbidden - see the lock ordering note in tiered_storage.go. Nomination +// therefore consults handle counts (which are atomic) instead of taking file +// locks, and workers use TryLock so they never block on user I/O. type lruQueue struct { - mu sync.Mutex - - nodeMap sync.Map - - wg sync.WaitGroup // tracks capacityChecker only - workerWg sync.WaitGroup // tracks worker goroutines - numWorkers int - activeWorkers int32 - + // mu guards head, tail and every field of every lruNode. + mu sync.Mutex head *lruNode tail *lruNode + // nodeMap indexes nodes by object name. It is only written under mu; it is + // a sync.Map so callers can test membership without taking mu. + nodeMap sync.Map - uploadChan chan string - doneChan chan struct{} - hallPassChan chan bool - - cachePath string - maxCacheSize float64 - totalUploadedSize int64 + // evictMu serialises eviction cycles so two cycles cannot nominate the same + // objects or interleave their batches. + evictMu sync.Mutex - threshold float64 - targetRatio float64 + checkerWg sync.WaitGroup + workerWg sync.WaitGroup + stopOnce sync.Once - tickerUnit time.Duration + uploadChan chan uploadJob + doneChan chan struct{} + cachePath string fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) - - //Functions to wire later into tiered_storage package - //upload function from tiered storage WIRE THIS LATER in tiered storage because we are using a function from there + size *cacheSizeTracker + + maxCacheSize float64 + // eviction starts above threshold*maxCacheSize and runs until usage reaches + // targetRatio*maxCacheSize, both fractions in (0,1] + threshold float64 + targetRatio float64 + numWorkers int + maxEviction uint32 + pollInterval time.Duration + + // uploads the object to cloud storage and removes the local copy. + // Supplied by TieredStorage; called with the object's file lock held. uploadandCleanFn func(name string) error - - // //delete locally - // localPath := filepath.Join(c.tmpPath, options.Name) - // c.mu.Lock() - // delete(c.fileMap, options.Name) - // c.mu.Unlock() - // os.Remove(localPath) - } func (q *lruQueue) StartPolicy() error { @@ -92,26 +138,54 @@ func (q *lruQueue) StartPolicy() error { if q.numWorkers <= 0 { return fmt.Errorf("lruQueue: numWorkers must be > 0") } - //initialize queue + if q.fileLocks == nil { + return fmt.Errorf("lruQueue: file locks not set") + } + if q.size == nil { + return fmt.Errorf("lruQueue: cache size tracker not set") + } + if q.pollInterval <= 0 { + q.pollInterval = capacityPollInterval + } + if q.maxEviction == 0 { + q.maxEviction = defaultMaxEviction + } + q.head = nil q.tail = nil - //channels q.doneChan = make(chan struct{}) - q.hallPassChan = make(chan bool, 1) - q.hallPassChan <- true + q.uploadChan = make(chan uploadJob, q.numWorkers*uploadQueueDepthPerWorker) - //go routines - q.wg.Add(1) + q.workerWg.Add(q.numWorkers) + for range q.numWorkers { + go q.worker() + } + + q.checkerWg.Add(1) go q.capacityChecker() return nil } +// StopPolicy shuts the policy down and waits for uploads already in flight. It +// does not drain the queue: local-only data is authoritative, so whatever has +// not been evicted stays in the cache for the next mount. func (q *lruQueue) StopPolicy() error { - close(q.doneChan) - // Wait for capacityChecker to exit — its deferred close(uploadChan) fires here, - // signalling workers that no more jobs are coming. - q.wg.Wait() + q.stopOnce.Do(func() { + if q.doneChan == nil { + return + } + close(q.doneChan) + q.checkerWg.Wait() + + // Block new eviction cycles before retiring the workers, so a + // concurrent EvictNow cannot send on a closed channel. + q.evictMu.Lock() + close(q.uploadChan) + q.evictMu.Unlock() + + q.workerWg.Wait() + }) return nil } @@ -119,62 +193,84 @@ func (q *lruQueue) Touch(name string) { q.Enqueue(name) } +// Enqueue makes name the most recently used object, adding it if it is new. +// A node that a worker owns is left alone; the worker re-queues it itself if +// the upload does not happen. func (q *lruQueue) Enqueue(name string) { - //Maybe have a duplicate , that touches essentially - - //lock earlier q.mu.Lock() defer q.mu.Unlock() - //search for new node first val, found := q.nodeMap.Load(name) - if found { - // Node already exists, no need to create a new one, just touch - node := val.(*lruNode) - q.extractNode(node) + if !found { + node := &lruNode{name: name} + q.nodeMap.Store(name, node) q.setHead(node) - } else { - // Node does not exist need to create a new one and put to top - newNode := &lruNode{name: name} - // if list is empty, set tail pointer to newNode - if q.tail == nil { - q.tail = newNode - } - q.nodeMap.Store(name, newNode) - q.setHead(newNode) + return + } + + node := val.(*lruNode) + if node.state != nodeQueued { + return } + // the object was used, so give it a clean slate on retries + node.failures = 0 + node.retryAfter = time.Time{} + q.extractNode(node) + q.setHead(node) } +// Dequeue removes name from the queue. If a worker is currently evicting the +// object then the eviction is cancelled instead, and the worker drops the node +// once it notices. func (q *lruQueue) Dequeue(name string) { - log.Trace("lruPolicy::removeNode : %s", name) + log.Trace("lruQueue::Dequeue : %s", name) q.mu.Lock() defer q.mu.Unlock() - val, found := q.nodeMap.LoadAndDelete(name) - if !found || val == nil { + val, found := q.nodeMap.Load(name) + if !found { return } node := val.(*lruNode) + if node.state == nodeEvicting { + node.state = nodeCancelled + return + } q.extractNode(node) + q.nodeMap.Delete(name) } +// mu must be held func (q *lruQueue) setHead(node *lruNode) { - // insert node at the head node.prev = nil node.next = q.head if q.head != nil { q.head.prev = node } q.head = node + if q.tail == nil { + q.tail = node + } } -func (q *lruQueue) extractNode(node *lruNode) { - // remove the node from its position in the list +// mu must be held +func (q *lruQueue) setTail(node *lruNode) { + node.next = nil + node.prev = q.tail + if q.tail != nil { + q.tail.next = node + } + q.tail = node + if q.head == nil { + q.head = node + } +} - // head case +// mu must be held +func (q *lruQueue) extractNode(node *lruNode) { if node == q.head { q.head = node.next } @@ -193,203 +289,274 @@ func (q *lruQueue) extractNode(node *lruNode) { node.next = nil } +func (q *lruQueue) stopping() bool { + select { + case <-q.doneChan: + return true + default: + return false + } +} + func (q *lruQueue) capacityChecker() { - defer q.wg.Done() - //defer close(q.uploadChan) + defer q.checkerWg.Done() - ticker := time.NewTicker(q.tickerUnit) + ticker := time.NewTicker(q.pollInterval) defer ticker.Stop() for { select { + case <-q.doneChan: + return case <-ticker.C: - // eviction - select { - case <-q.hallPassChan: - - //1. Check if we need eviction - curSize, err := common.GetUsage(q.cachePath) - if err != nil { - log.Err("lruPolicy::capacityChecker : failed to get usage: %v", err) - q.hallPassChan <- true - continue - } - if curSize/q.maxCacheSize <= q.threshold { - q.hallPassChan <- true - break - } - //targetRatio should always be less than thresholdRatio - - //2. Find difference to evict down to target ratio - difference := curSize - q.maxCacheSize*q.targetRatio - curEvictedSpace := 0 - actualEvictedSpace := 0 - atomic.StoreInt64(&q.totalUploadedSize, 0) - evicFail := false - - //3. LRU Eviction to match difference - for actualEvictedSpace < int(difference) && !evicFail { - // Start nomination from what's already been confirmed uploaded, - // so we only nominate enough new files to cover the remaining gap. - curEvictedSpace = actualEvictedSpace - - //initialize channel - q.uploadChan = make(chan string, 1000) - - //start workers here - q.workerWg.Add(q.numWorkers) - for i := 0; i < q.numWorkers; i++ { - go q.worker() - } - - //populate upload channel for workers - for curEvictedSpace < int(difference) { - nodeSize, evicted := q.eviction() - if !evicted { - evicFail = true - break - } - curEvictedSpace += int(nodeSize) - } - - //close upload chan and wait for workers to process all remaining jobs - close(q.uploadChan) - q.workerWg.Wait() - - //update the actual evicted space here - actualEvictedSpace = int(q.totalUploadedSize) - - //if done is closed we need to check shutdown to prevent infinite loop - select { - case <-q.doneChan: - return - default: - } - - //we can also have a case here if no files were evicted then we can break on this tick if we so choose - - } - //give hall pass back when actual evicted space is satisfied - q.hallPassChan <- true - - case <-q.doneChan: - return - - //default return? - default: + q.size.Reconcile() + if float64(q.size.Used()) <= q.maxCacheSize*q.threshold { + continue } - case <-q.doneChan: + q.evictDownTo(int64(q.maxCacheSize * q.targetRatio)) + } + } +} + +// EvictNow synchronously makes room for growBytes more bytes of local data and +// reports whether the cache has room afterwards. Callers may hold a file lock: +// workers never block on file locks, so an object cannot deadlock against its +// own eviction. +func (q *lruQueue) EvictNow(growBytes int64) bool { + target := min(int64(q.maxCacheSize*q.targetRatio), int64(q.maxCacheSize)-growBytes) + q.evictDownTo(target) + return float64(q.size.Used()+growBytes) <= q.maxCacheSize +} + +// evictDownTo uploads least-recently-used objects until usage reaches +// targetUsed. It gives up as soon as a pass evicts nothing, which is what +// happens when every remaining object is open, in use, or failing to upload. +func (q *lruQueue) evictDownTo(targetUsed int64) { + q.evictMu.Lock() + defer q.evictMu.Unlock() + + if q.stopping() { + return + } + + // Work from a local estimate rather than re-reading the tracker each pass: + // the tracker is updated by the upload callback, so re-reading it would + // couple this loop to whenever that callback happens to run. + used := q.size.Used() + + for pass := range evictionPassLimit { + if used <= targetUsed { + return + } + + names := q.nominate(used-targetUsed, int(q.maxEviction)) + if len(names) == 0 { + log.Info("lruQueue::evictDownTo : nothing is eligible for eviction") + return + } + + freed, evicted := q.runBatch(names) + used -= freed + + if evicted == 0 { + log.Warn( + "lruQueue::evictDownTo : pass %d evicted nothing (using %d, target %d)", + pass, + used, + targetUsed, + ) return } } } -func (q *lruQueue) eviction() (int64, bool) { - q.mu.Lock() - nodeToEvict := q.tail - if nodeToEvict == nil { - q.mu.Unlock() - return 0, false +// runBatch hands every name to the worker pool and waits for that batch to +// finish. It returns the bytes and the number of objects actually evicted. +func (q *lruQueue) runBatch(names []string) (int64, int32) { + var wg sync.WaitGroup + var freed atomic.Int64 + var evicted atomic.Int32 + + wg.Add(len(names)) + for i, name := range names { + select { + case q.uploadChan <- uploadJob{name: name, wg: &wg, freed: &freed, evicted: &evicted}: + case <-q.doneChan: + // shutting down - hand back everything we could not dispatch + for _, undelivered := range names[i:] { + q.requeue(undelivered, false) + wg.Done() + } + wg.Wait() + return freed.Load(), evicted.Load() + } } + wg.Wait() + + return freed.Load(), evicted.Load() +} + +// nominate detaches least-recently-used nodes until their combined size covers +// bytesNeeded, marking each as owned by a worker. Objects with open handles are +// promoted to the head instead: they are by definition in use, and promoting +// them stops the next pass from considering them again. +// +// Each candidate is measured here, under mu. That is a syscall per nominated +// object while the queue is locked, but the count is bounded by the deficit +// being covered, and it keeps nomination from having to reach back into the +// component's file map. +func (q *lruQueue) nominate(bytesNeeded int64, limit int) []string { + q.mu.Lock() + defer q.mu.Unlock() + + now := time.Now() + var names []string + var busy []*lruNode + var total int64 + + for node := q.tail; node != nil && len(names) < limit && total < bytesNeeded; { + // remember where to go next before the node is unlinked + prev := node.prev + + switch { + case node.state != nodeQueued: + // a worker owns it; it should not be in the list at all - //1. loop through and find the first applicable node - for nodeToEvict != nil { - prevNode := nodeToEvict.prev + case now.Before(node.retryAfter): + // backing off after a failed upload - flock := q.fileLocks.Get(nodeToEvict.name) - flock.RLock() - handleCount := flock.Count() - flock.RUnlock() + case q.fileLocks.Get(node.name).Count() > 0: + busy = append(busy, node) - if handleCount == 0 { - break + default: + info, err := os.Stat(filepath.Join(q.cachePath, node.name)) + if err != nil { + // there is no local copy left to evict + log.Warn("lruQueue::nominate : dropping %s [%v]", node.name, err) + q.extractNode(node) + q.nodeMap.Delete(node.name) + break + } + q.extractNode(node) + node.state = nodeEvicting + names = append(names, node.name) + total += info.Size() } - //node has open handles touch node - q.extractNode(nodeToEvict) - q.setHead(nodeToEvict) - nodeToEvict = prevNode + node = prev } - //all files are in use - if nodeToEvict == nil { - q.mu.Unlock() - return 0, false + // Promote busy nodes after the walk, not during it: re-linking a node while + // traversing can lead the walk back over nodes it already visited. The list + // is walked oldest-first, so promoting in that order preserves their + // relative recency. + for _, node := range busy { + q.extractNode(node) + q.setHead(node) } - name := nodeToEvict.name - //2. Remove file from queue so not accidentally chosen again - q.extractNode(nodeToEvict) - q.nodeMap.Delete(name) + return names +} - q.mu.Unlock() +// claimed reports whether the worker may still upload name. +func (q *lruQueue) claimed(name string) bool { + q.mu.Lock() + defer q.mu.Unlock() - //3. Get the node size that we evict - localPath := filepath.Join(q.cachePath, name) + val, found := q.nodeMap.Load(name) + return found && val.(*lruNode).state == nodeEvicting +} - fileInfo, err := os.Stat(localPath) - if err != nil { - log.Err("lruPolicy::capacityChecker : failed to stat file: %v", err) - return 0, false +// requeue returns a nominated node to the queue. A failed upload goes to the +// tail with a backoff, so one unwritable object cannot monopolise the workers. +// An object that was merely busy goes to the head, because being in use is what +// the head of an LRU means. +func (q *lruQueue) requeue(name string, failed bool) { + q.mu.Lock() + defer q.mu.Unlock() + + val, found := q.nodeMap.Load(name) + if !found { + return } - nodeSize := fileInfo.Size() - //4. Send node to channel to be uploaded by workers - select { - case q.uploadChan <- name: - case <-q.doneChan: - return 0, false + node := val.(*lruNode) + if node.state == nodeCancelled { + q.nodeMap.Delete(name) + return + } + + node.state = nodeQueued + if failed { + node.failures++ + node.retryAfter = time.Now().Add(retryBackoff(node.failures)) + q.setTail(node) + return + } + q.setHead(node) +} + +// drop forgets a node whose local copy no longer exists. +func (q *lruQueue) drop(name string) { + q.mu.Lock() + defer q.mu.Unlock() + + if val, found := q.nodeMap.Load(name); found { + q.extractNode(val.(*lruNode)) } + q.nodeMap.Delete(name) +} - return nodeSize, true +func retryBackoff(failures uint32) time.Duration { + return min(time.Second*time.Duration(1< 0 { + q.requeue(job.name, false) + return } + + info, err := os.Stat(filepath.Join(q.cachePath, job.name)) + if err != nil { + log.Warn("lruQueue::evictOne : %s has no local copy, dropping it [%v]", job.name, err) + q.drop(job.name) + return + } + + if err := q.uploadandCleanFn(job.name); err != nil { + log.Err("lruQueue::evictOne : %s upload failed [%v]", job.name, err) + q.requeue(job.name, true) + return + } + + job.freed.Add(info.Size()) + job.evicted.Add(1) + q.drop(job.name) } diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index d154f6ce5..7b1b415f3 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -71,8 +71,11 @@ func (suite *lruPolicyTestSuite) setupTestHelper( threshold: threshold, targetRatio: targetRatio, numWorkers: numWorkers, - tickerUnit: time.Millisecond, - fileLocks: common.NewLockMap(), // required by eviction() and worker() + pollInterval: time.Millisecond, + fileLocks: common.NewLockMap(), // required by nominate() and evictOne() + // reconcile on every tick: these tests create files directly on disk, + // so the tracker has no incremental updates to work from + size: newCacheSizeTracker(cachePath, 0), uploadandCleanFn: func(name string) error { return nil diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 9fe0eb053..2c21fc31e 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -27,11 +27,14 @@ package tiered_storage import ( "context" + "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "sync" + "sync/atomic" "syscall" "time" @@ -42,18 +45,33 @@ import ( "github.com/Seagate/cloudfuse/internal/handlemap" ) -/* NOTES: - - Component shall have a structure which inherits "internal.BaseComponent" to participate in pipeline - - Component shall register a name and its constructor to participate in pipeline (add by default by generator) - - Order of calls : Constructor -> Configure -> Start ..... -> Stop - - To read any new setting from config file follow the Configure method default comments +/* + TieredStorage treats local storage as the authoritative tier and cloud + storage as an overflow tier behind it. + + - Files created through the mount live only on local disk. They move to the + cloud when the LRU evicts them, and the local copy is removed at that point. + - Objects that are already in the cloud are cached locally on open and the + local copy is dropped on last close, after any changes are uploaded. Data is + therefore resident in both tiers for as short a time as possible. + + Lock ordering. Acquire in this order and release in reverse: + + 1. the object's file lock (c.fileLocks) - at most one, except rename, which + takes source and destination in lexical order + 2. lruQueue.evictMu + 3. lruQueue.mu + + Never take a file lock while holding lruQueue.mu. The eviction path obeys this + by reading handle counts (which are atomic) during nomination and by using + TryLock in its workers, so it never blocks on user I/O. */ // Common structure for Component type TieredStorage struct { internal.BaseComponent - //fileMap map[string]*FileNode - fileMap sync.Map + + fileMap sync.Map // uses object name (common.JoinUnixFilepath) policy *lruQueue @@ -61,29 +79,17 @@ type TieredStorage struct { fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) tmpPath string // uses os.Separator (filepath.Join) - // Still need mutex to protect fileMap and lruQueue - mu sync.Mutex - + cacheSize *cacheSizeTracker maxCacheSize float64 } -// define a file node structure to hold file related information +// FileNode tracks file state. Its atomic fields can be accessed concurrently. +// Name changes can only be done while holding flock. type FileNode struct { name string - size uint64 - prev *FileNode - next *FileNode - cloudBacked bool - isDirty bool - // Add more attributes as needed, e.g., last accessed time, etc. -} - -// Add more attributes as needed, e.g., last accessed time, etc. -type LRUQueue struct { - head *FileNode - tail *FileNode - maxSize uint64 //figure this out later based on config or some heuristics - currentSize uint64 + size atomic.Int64 + cloudBacked atomic.Bool + isDirty atomic.Bool } // Structure defining your config parameters @@ -94,9 +100,15 @@ type TieredStorageOptions struct { } const ( - compName = "tiered_storage" - defaultMaxEviction = 000000 //placeholder until we figure out - + compName = "tiered_storage" + // TODO: make thresholds configurable + defaultHighThreshold = 0.8 + defaultLowThreshold = 0.6 + defaultParallelism = 8 + defaultMaxEviction = 5000 + capacityPollInterval = time.Second + reconcileCapacityInterval = 5 * time.Minute + partialDownloadSuffix = ".cloudfuse-partial" ) // Verification to check satisfaction criteria with Component Interface @@ -120,9 +132,14 @@ func (c *TieredStorage) SetNextComponent(nc internal.Component) { func (c *TieredStorage) Start(ctx context.Context) error { log.Trace("TieredStorage::Start : Starting component %s", c.Name()) - // TieredStorage : start code goes here + // A crash can leave partial downloads behind. They are not valid object + // data, so remove them before anything can list, open or upload them. + c.removePartialDownloads() + + // Seed the usage counter from what is actually on disk. This is the one + // place where measuring the whole directory is worth its cost. + c.cacheSize.Refresh() - //Start the policy if c.policy != nil { if err := c.policy.StartPolicy(); err != nil { log.Err("TieredStorage::Start : failed to start LRU policy [%v]", err) @@ -134,6 +151,9 @@ func (c *TieredStorage) Start(ctx context.Context) error { } // Stop : Stop the component functionality and kill all threads started +// +// The local cache is deliberately left in place: for this component it holds +// the only copy of any data that has not been evicted yet. func (c *TieredStorage) Stop() error { log.Trace("TieredStorage::Stop : Stopping component %s", c.Name()) @@ -144,21 +164,43 @@ func (c *TieredStorage) Stop() error { return nil } +// removePartialDownloads deletes interrupted downloads left by a previous run. +func (c *TieredStorage) removePartialDownloads() { + err := filepath.WalkDir(c.tmpPath, func(path string, d fs.DirEntry, err error) error { + if err != nil || d == nil || d.IsDir() { + return nil //nolint:nilerr // an unreadable entry is not worth aborting the sweep + } + if filepath.Ext(path) != partialDownloadSuffix { + return nil + } + log.Info("TieredStorage::removePartialDownloads : removing %s", path) + if rmErr := os.Remove(path); rmErr != nil { + log.Warn( + "TieredStorage::removePartialDownloads : %s remove failed [%v]", + path, + rmErr, + ) + } + return nil + }) + if err != nil { + log.Warn("TieredStorage::removePartialDownloads : %s walk failed [%v]", c.tmpPath, err) + } +} + // Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself // // Return failure if any config is not valid to exit the process func (c *TieredStorage) Configure(_ bool) error { log.Trace("TieredStorage::Configure : %s", c.Name()) - // >> If you do not need any config parameters remove below code and return nil conf := TieredStorageOptions{} err := config.UnmarshalKey(c.Name(), &conf) if err != nil { log.Err("TieredStorage::Configure : config error [invalid config attributes]") return fmt.Errorf("TieredStorage: config error [invalid config attributes]") } - // Extract values from 'conf' and store them as you wish here - // CLAUDE GENERATED HERE, CAUSE I HAD NO CLUE + c.tmpPath = filepath.Clean(common.ExpandPath(conf.TmpPath)) if c.tmpPath == "" || c.tmpPath == "." { return fmt.Errorf("TieredStorage: path not set in config") @@ -170,17 +212,19 @@ func (c *TieredStorage) Configure(_ bool) error { return fmt.Errorf("TieredStorage: failed to create tmp path: %w", err) } - //figure out the maxCache size stuff here, there is just a bunch of configure stuff that we need to figure out - c.maxCacheSize = conf.MaxSizeMB * 1024 * 1024 - //Wire in the LRU Policy + c.maxCacheSize = conf.MaxSizeMB * common.MbToBytes + c.cacheSize = newCacheSizeTracker(c.tmpPath, reconcileCapacityInterval) + c.policy = &lruQueue{ cachePath: c.tmpPath, maxCacheSize: c.maxCacheSize, fileLocks: c.fileLocks, - threshold: 0.8, - targetRatio: 0.6, - numWorkers: 8, - tickerUnit: time.Millisecond, + size: c.cacheSize, + threshold: defaultHighThreshold, + targetRatio: defaultLowThreshold, + numWorkers: defaultParallelism, + maxEviction: defaultMaxEviction, + pollInterval: capacityPollInterval, uploadandCleanFn: c.uploadandCleanFile, } @@ -226,10 +270,8 @@ func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { func (c *TieredStorage) createFileUnlocked( options internal.CreateFileOptions, ) (*handlemap.Handle, error) { - if c.isOverLocalLimit(0, options.Name, "create") { - return nil, fmt.Errorf("cache limit exceeded, cannot create file") - //eventually put a eviction here - } + // A new file holds no data yet, so there is nothing to make room for. + // WriteFile reserves space as the file grows. //Create the file in the local cache, we will ignore the create empty and cloud stuff for now localPath := filepath.Join(c.tmpPath, options.Name) @@ -250,16 +292,8 @@ func (c *TieredStorage) createFileUnlocked( } //Add file node to file map with cloudBacked as false - node := &FileNode{ - name: options.Name, - size: uint64(0), - cloudBacked: false, - isDirty: true, - } - // c.mu.Lock() - // c.fileMap[options.Name] = node - // c.mu.Unlock() - + node := &FileNode{name: options.Name} + node.isDirty.Store(true) c.fileMap.Store(options.Name, node) //create handle @@ -289,46 +323,32 @@ func (c *TieredStorage) CreateFile( func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { log.Trace("TieredStorage::DeleteFile : name=%s", options.Name) - //Lock the file first + flock := c.fileLocks.Get(options.Name) flock.Lock() + defer flock.Unlock() - //Unlock manually so we only hold one lock at a time - //defer flock.Unlock() - + // Read the state only after taking the lock: an eviction worker may have + // been uploading this object right up until we acquired it. val, exists := c.fileMap.Load(options.Name) - //Potential local or local + cloud state - if exists { - node := val.(*FileNode) - localPath := filepath.Join(c.tmpPath, options.Name) - - //Local and Cloud State - if node.cloudBacked { - //Both local and cloud state - //delete from cloud first - err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) - if err != nil { - flock.Unlock() - return err - } - } - //Local only State - //remove from LRU if it is in there already and delete local file - c.fileMap.Delete(options.Name) - flock.Unlock() - c.policy.Dequeue(options.Name) - os.Remove(localPath) + if !exists { + // cloud only + return c.NextComponent().DeleteFile(options) + } - //Cloud only state - } else { - //delete from cloud - flock.Unlock() - err := c.NextComponent().DeleteFile(internal.DeleteFileOptions{Name: options.Name}) - if err != nil { + node := val.(*FileNode) + if node.cloudBacked.Load() { + if err := c.NextComponent().DeleteFile(options); err != nil { return err } } - return nil + + // Cancel any eviction of this object before dropping our own state, so a + // worker cannot resurrect it in the cloud after the user deleted it. + c.policy.Dequeue(options.Name) + c.fileMap.Delete(options.Name) + + return c.purgeLocal(options.Name) } // OpenFile: Makes the file available in the local cache for further file operations. @@ -369,11 +389,8 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H options.Name, ) //Read from local disk, create file node and add to file map - node := &FileNode{ - name: options.Name, - size: uint64(info.Size()), - cloudBacked: false, - } + node := &FileNode{name: options.Name} + node.size.Store(info.Size()) c.fileMap.Store(options.Name, node) } else { @@ -387,15 +404,13 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H return nil, err } // file exists in cloud, create local copy (name doesn't matter)and add to file map - localCopyNode := &FileNode{ - name: options.Name, - size: uint64(info.Size), - cloudBacked: true, - } - // check if we are over the local cache limit - if c.isOverLocalLimit(uint64(info.Size), options.Name, "open") { - // we are over the local cache limit, return error for now, - return nil, fmt.Errorf("cache limit exceeded, cannot open file") + localCopyNode := &FileNode{name: options.Name} + localCopyNode.size.Store(info.Size) + localCopyNode.cloudBacked.Store(true) + + // make room for the copy we are about to download + if err := c.reserveSpace(info.Size); err != nil { + return nil, err } //download it to the local cache and add to file map err = c.downloadCopyFromCloud(options) @@ -434,78 +449,124 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H return handle, nil } -// openFileHelper : function to download copy from cloud and add to local cache +// downloadCopyFromCloud caches a cloud object on local storage. +// +// The data lands in a temporary file and is renamed into place only once it is +// complete. A crash can therefore never leave a truncated file at the object's +// real path, which matters because local data is authoritative here: a partial +// download found at a real path would later be uploaded over the good cloud copy. func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) error { - //create folder if not exists, wait check what 0755 does localPath := filepath.Join(c.tmpPath, options.Name) err := os.MkdirAll(filepath.Dir(localPath), 0755) if err != nil { return err } - //Open temporary download handle to the local file path - localFileHandle, err := common.OpenFile( - localPath, + + partPath := fmt.Sprintf("%s.%d%s", localPath, os.Getpid(), partialDownloadSuffix) + partFile, err := common.OpenFile( + partPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, options.Mode, ) if err != nil { return err } - defer localFileHandle.Close() //Download err = c.NextComponent().CopyToFile(internal.CopyToFileOptions{ Name: options.Name, Offset: 0, Count: 0, - File: localFileHandle, + File: partFile, }) + if err == nil { + err = partFile.Sync() + } + if closeErr := partFile.Close(); err == nil { + err = closeErr + } if err != nil { - _ = os.Remove(localPath) + log.Err("TieredStorage::downloadCopyFromCloud : %s download failed [%v]", options.Name, err) + _ = os.Remove(partPath) return err } + + if err := replaceFile(partPath, localPath); err != nil { + log.Err("TieredStorage::downloadCopyFromCloud : %s rename failed [%v]", options.Name, err) + _ = os.Remove(partPath) + return err + } + + if info, statErr := os.Stat(localPath); statErr == nil { + c.cacheSize.Add(info.Size()) + } + //some sort of mode handling return nil } -// rough rough rough implementation of checking limit of cache, -// need to figure out eviction and other details before finalizing -func (c *TieredStorage) isOverLocalLimit( - newFileSize uint64, - fileName string, - requestType string, -) bool { - - if c.maxCacheSize == 0 { - // if maxCacheSize is 0, it means there is no limit on local cache size, so we can return false - return false +// replaceFile moves src over dst. Windows will not replace a destination that +// another handle holds without share-delete, so fall back to removing it first. +func replaceFile(src, dst string) error { + err := os.Rename(src, dst) + if err == nil { + return nil } - - //find ExistingSize of file if exists - existingSize := uint64(0) - if val, ok := c.fileMap.Load(fileName); ok { - existingSize = val.(*FileNode).size + if rmErr := os.Remove(dst); rmErr != nil && !errors.Is(rmErr, os.ErrNotExist) { + return err } + return os.Rename(src, dst) +} - addedFileSize := int64(newFileSize) - int64(existingSize) +// purgeLocal removes an object's local copy and updates the usage counter. +// The object's file lock must be held. +func (c *TieredStorage) purgeLocal(name string) error { + localPath := filepath.Join(c.tmpPath, name) - //if we didn't modify the size of the file then - if addedFileSize <= 0 { - return false + info, statErr := os.Stat(localPath) + err := os.Remove(localPath) + if err == nil && statErr == nil { + c.cacheSize.Add(-info.Size()) } + return err +} - //get current cache size - currSize, err := common.GetUsage(c.tmpPath) - if err != nil { - log.Err("TieredStorage::IsOverLocalLimit : failed to get current cache size [%v]", err) - return false +// recordSize updates what we believe a cached file's size to be and adjusts the +// usage counter by the difference. +func (c *TieredStorage) recordSize(node *FileNode, size int64) { + if node == nil { + return } + c.cacheSize.Resize(node.size.Swap(size), size) +} - if float64(currSize)+float64(addedFileSize) > (c.maxCacheSize + 4096) { - //should include some error message - return true +// reserveSpace makes room for growBytes more bytes of local data. +// +// Running out of local space is not an error in this component: cloud storage +// is the overflow tier, so the first response is to evict. ENOSPC is only the +// right answer when nothing can be evicted - every object is open, or uploads +// are failing. +// +// Callers may hold a file lock. Eviction workers never block on file locks, so +// the object being written cannot deadlock against its own eviction. +func (c *TieredStorage) reserveSpace(growBytes int64) error { + if c.maxCacheSize <= 0 || growBytes <= 0 { + return nil } - return false + if float64(c.cacheSize.Used()+growBytes) <= c.maxCacheSize { + return nil + } + if c.policy == nil || c.policy.EvictNow(growBytes) { + return nil + } + + log.Err( + "TieredStorage::reserveSpace : cache is full and nothing can be evicted (need %d bytes, using %d of %d)", + growBytes, + c.cacheSize.Used(), + int64(c.maxCacheSize), + ) + return syscall.ENOSPC } func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int, error) { @@ -527,23 +588,31 @@ func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int } func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { - //1.Get the file opbject + //1.Get the file object f := options.Handle.GetFileObject() if f == nil { return 0, syscall.EBADF } - //2. Check if exceeds limits - newSize := options.Offset + int64(len(options.Data)) - if c.isOverLocalLimit(uint64(newSize), options.Handle.Path, "write") { - return 0, syscall.ENOSPC - //eventually put eviction here + var node *FileNode + if val, ok := c.fileMap.Load(options.Handle.Path); ok { + node = val.(*FileNode) + } + + //2. Make room for however much bigger this write makes the file + appending := options.Handle.Flags.IsSet(handlemap.HandleOpenedAppend) + growth := int64(len(options.Data)) + if !appending && node != nil { + growth = max(options.Offset+int64(len(options.Data))-node.size.Load(), 0) + } + if err := c.reserveSpace(growth); err != nil { + return 0, err } //3. Decide where to write in file var bytesWritten int var err error - if options.Handle.Flags.IsSet(handlemap.HandleOpenedAppend) { + if appending { //write to end of file, standard bytesWritten, err = f.Write(options.Data) } else { @@ -554,13 +623,14 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro //4. Mark file as dirty for release later if err == nil { c.setHandleDirty(options.Handle) - //update file node size in file map - if val, ok := c.fileMap.Load(options.Handle.Path); ok { - node := val.(*FileNode) - node.size = uint64(newSize) - node.isDirty = true + if node != nil { + node.isDirty.Store(true) + // One fstat is the cheapest way to be right about the new size for + // every kind of write - appending, sparse, or overwriting in place. + if info, statErr := f.Stat(); statErr == nil { + c.recordSize(node, info.Size()) + } } - } else { log.Err( "TieredStorage::WriteFile : failed to write %s [%s]", @@ -613,9 +683,7 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released flock := c.fileLocks.Get(options.Handle.Path) flock.Lock() - - //Ok we have to manually unlock the file now instead - //defer flock.Unlock() + defer flock.Unlock() //Dec Handle Count First flock.Dec() @@ -632,55 +700,50 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { //remove from global handle map handlemap.Delete(options.Handle.ID) - //Check if this is the last file handle - handleCount := flock.Count() + //Only the last handle decides what happens to the local copy + if flock.Count() > 0 { + return nil + } + + val, ok := c.fileMap.Load(options.Handle.Path) + if !ok { + // the object was deleted or evicted while it was open - closing a + // deleted file is normal, so this is not an error + log.Debug( + "TieredStorage::ReleaseFile : %s has no local data left", + options.Handle.Path, + ) + return nil + } + node := val.(*FileNode) - //it is the last handle - if handleCount == 0 { - //is file cloudbacked - val, ok := c.fileMap.Load(options.Handle.Path) - if !ok { + if !node.cloudBacked.Load() { + // Local-only data is authoritative. The LRU decides when it moves to + // cloud storage, and the local copy stays until it does. + c.policy.Enqueue(options.Handle.Path) + return nil + } + + if node.isDirty.Load() { + if err := c.uploadCachedFile(options.Handle.Path); err != nil { + // Keep the local copy: it is newer than the cloud object. Hand it + // to the LRU so the upload is retried, rather than stranding the + // only good copy of the data in the cache with nothing watching it. log.Err( - "TieredStorage::ReleaseFile : internal error: file %s not found in map", + "TieredStorage::ReleaseFile : upload failed for %s [%v]", options.Handle.Path, + err, ) - flock.Unlock() - return syscall.EBADF - } - node := val.(*FileNode) - if node.cloudBacked { - //File was modified - if node.isDirty { - //Upload - err := c.uploadCachedFile(options.Handle.Path) - if err != nil { - log.Err( - "TieredStorage::ReleaseFile : upload failed for %s [%v]", - options.Handle.Path, - err, - ) - flock.Unlock() - return err - } - } - //Whether File was modified or not, delete local file copy - localPath := filepath.Join(c.tmpPath, options.Handle.Path) - c.fileMap.Delete(options.Handle.Path) - os.Remove(localPath) - flock.Unlock() - } else { - // update LRU add to queue because cleaning up the file should be handled once the file is uploaded in LRU policy logic - //Unlock the file first so we don't hold two locks at the same time - flock.Unlock() - if c.policy != nil { - c.policy.Enqueue(options.Handle.Path) - } + c.policy.Enqueue(options.Handle.Path) + return err } - } else { - //if not the last handle - flock.Unlock() + node.isDirty.Store(false) } - return nil + + // Cached cloud data is dropped as soon as the last handle closes, so an + // object spends as little time as possible resident in both tiers. + c.fileMap.Delete(options.Handle.Path) + return c.purgeLocal(options.Handle.Path) } func (c *TieredStorage) uploadCachedFile(name string) error { @@ -708,14 +771,16 @@ func (c *TieredStorage) uploadCachedFile(name string) error { return uploadErr } +// uploadandCleanFile moves an object to cloud storage and removes the local +// copy. It is the LRU's eviction callback, so it runs with the object's file +// lock held. func (c *TieredStorage) uploadandCleanFile(name string) error { err := c.uploadCachedFile(name) if err != nil { return err } - localPath := filepath.Join(c.tmpPath, name) c.fileMap.Delete(name) - err = os.Remove(localPath) + err = c.purgeLocal(name) if err != nil { log.Err("TieredStorage::uploadandCleanFile : %s remove failed [%v]", name, err) return err @@ -750,7 +815,7 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { if exists { node := val.(*FileNode) // //Local and Cloud State - if node.cloudBacked { + if node.cloudBacked.Load() { //just rename from the cloud err := c.NextComponent().RenameFile(options) if err != nil { @@ -770,10 +835,12 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { node.name = options.Dst c.fileMap.Store(options.Dst, node) - //Check if it is in the LRU first - _, inLRU := c.policy.nodeMap.Load(options.Src) - if inLRU { - c.policy.Dequeue(options.Src) + // Move the queue entry across. The source is still in nodeMap even if a + // worker is mid-eviction, and Dequeue cancels that eviction - which is + // what stops a renamed object from falling out of the queue entirely. + _, wasQueued := c.policy.nodeMap.Load(options.Src) + c.policy.Dequeue(options.Src) + if wasQueued { c.policy.Enqueue(options.Dst) } //Change the handle and the lock counts diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 25e381cc2..47b991537 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -381,7 +381,7 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudNoDirtyFile() { val, ok := suite.tieredStorage.fileMap.Load(path) node := val.(*FileNode) - suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(node.cloudBacked.Load(), "File should be marked as cloud-backed") suite.assert.True(ok, "File should be tracked in the fileMap") //File should be "cloudBacked" and not dirty so on release the file should be deleted from local and the handle clean @@ -436,7 +436,7 @@ func (suite *tieredStorageTestSuite) TestReleaseCloudDirtyFile() { val, exists := suite.tieredStorage.fileMap.Load(path) node := val.(*FileNode) - suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(node.cloudBacked.Load(), "File should be marked as cloud-backed") suite.assert.True(exists, "File should be tracked in the fileMap") //File should be "cloudBacked" and dirty so on release the file should be deleted from local and the handle clean @@ -548,7 +548,7 @@ func (suite *tieredStorageTestSuite) TestWriteReadDirtyState() { val, exists := suite.tieredStorage.fileMap.Load(path) node := val.(*FileNode) - suite.assert.True(node.cloudBacked, "File should be marked as cloud-backed") + suite.assert.True(node.cloudBacked.Load(), "File should be marked as cloud-backed") suite.assert.True(exists, "File should be tracked in the fileMap") //1. Write to handle @@ -597,7 +597,7 @@ func (suite *tieredStorageTestSuite) TestWriteReadDirtyState() { val, _ = suite.tieredStorage.fileMap.Load(path) node = val.(*FileNode) - suite.assert.True(node.isDirty, "File should be marked as dirty") + suite.assert.True(node.isDirty.Load(), "File should be marked as dirty") //4. Release the read should upload to cloud err = suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle2}) @@ -649,7 +649,7 @@ func (suite *tieredStorageTestSuite) TestReleaseLocalToLRUQueue() { val, exists := suite.tieredStorage.fileMap.Load(path) node := val.(*FileNode) - suite.assert.False(node.cloudBacked, "File should not be marked as cloud-backed") + suite.assert.False(node.cloudBacked.Load(), "File should not be marked as cloud-backed") suite.assert.True(exists, "File should be tracked in the fileMap") // 2. Release this local file From 403a1f6cef78ecdec4c65350250ff376b7c25e9c Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 16:46:29 -0600 Subject: [PATCH 48/89] Comments --- component/tiered_storage/lru_policy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 2a797a03b..65aea2c2c 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -475,12 +475,14 @@ func (q *lruQueue) requeue(name string, failed bool) { q.mu.Lock() defer q.mu.Unlock() + // get the node val, found := q.nodeMap.Load(name) if !found { return } - node := val.(*lruNode) + + // was the file deleted or renamed? if node.state == nodeCancelled { q.nodeMap.Delete(name) return From 6f04abca27ad5206f9312242ebe88d1940b50657 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 16:58:09 -0600 Subject: [PATCH 49/89] Don't retry eviction --- component/tiered_storage/lru_policy.go | 104 +++++------------- component/tiered_storage/lru_policy_test.go | 55 +++++++++ .../tiered_storage/tiered_storage_test.go | 10 +- 3 files changed, 92 insertions(+), 77 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 65aea2c2c..facd8f405 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -29,7 +29,6 @@ import ( "os" "path/filepath" "sync" - "sync/atomic" "time" "github.com/Seagate/cloudfuse/common" @@ -37,14 +36,6 @@ import ( ) const ( - // evictionPassLimit caps how many nominate-and-wait rounds one eviction - // cycle may run, so a cache that cannot be drained does not spin. - evictionPassLimit = 8 - - // maxRetryBackoff bounds the delay applied to an object whose upload keeps - // failing. - maxRetryBackoff = 5 * time.Minute - // uploadQueueDepthPerWorker sizes the job channel. It only needs enough // slack that the dispatcher is not serialised against the workers. uploadQueueDepthPerWorker = 16 @@ -72,19 +63,13 @@ type lruNode struct { name string state nodeState - // consecutive failed uploads, and the time before which this object should - // not be nominated again - failures uint32 - retryAfter time.Time } -// uploadJob is one nominated object. The waitgroup and counters belong to the -// eviction pass that nominated it, so a pass waits only for its own batch. +// uploadJob is one nominated object. The waitgroup belongs to the eviction +// pass that nominated it, so a pass waits only for its own batch. type uploadJob struct { - name string - wg *sync.WaitGroup - freed *atomic.Int64 - evicted *atomic.Int32 + name string + wg *sync.WaitGroup } // lruQueue decides which local-only objects move to cloud storage. @@ -212,9 +197,6 @@ func (q *lruQueue) Enqueue(name string) { if node.state != nodeQueued { return } - // the object was used, so give it a clean slate on retries - node.failures = 0 - node.retryAfter = time.Time{} q.extractNode(node) q.setHead(node) } @@ -328,9 +310,9 @@ func (q *lruQueue) EvictNow(growBytes int64) bool { return float64(q.size.Used()+growBytes) <= q.maxCacheSize } -// evictDownTo uploads least-recently-used objects until usage reaches -// targetUsed. It gives up as soon as a pass evicts nothing, which is what -// happens when every remaining object is open, in use, or failing to upload. +// evictDownTo runs one eviction pass, nominating enough least-recently-used +// objects to reach targetUsed. If the pass does not bring usage below the high +// threshold, a later capacity check may try again. func (q *lruQueue) evictDownTo(targetUsed int64) { q.evictMu.Lock() defer q.evictMu.Unlock() @@ -339,48 +321,36 @@ func (q *lruQueue) evictDownTo(targetUsed int64) { return } - // Work from a local estimate rather than re-reading the tracker each pass: - // the tracker is updated by the upload callback, so re-reading it would - // couple this loop to whenever that callback happens to run. used := q.size.Used() + if used <= targetUsed { + return + } - for pass := range evictionPassLimit { - if used <= targetUsed { - return - } - - names := q.nominate(used-targetUsed, int(q.maxEviction)) - if len(names) == 0 { - log.Info("lruQueue::evictDownTo : nothing is eligible for eviction") - return - } - - freed, evicted := q.runBatch(names) - used -= freed + names := q.nominate(used-targetUsed, int(q.maxEviction)) + if len(names) == 0 { + log.Info("lruQueue::evictDownTo : nothing is eligible for eviction") + return + } - if evicted == 0 { - log.Warn( - "lruQueue::evictDownTo : pass %d evicted nothing (using %d, target %d)", - pass, - used, - targetUsed, - ) - return - } + q.runBatch(names) + if usedAfter := q.size.Used(); float64(usedAfter) > q.maxCacheSize*q.threshold { + log.Err( + "lruQueue::evictDownTo : usage remains above high threshold after eviction (using %d, high threshold %.0f)", + usedAfter, + q.maxCacheSize*q.threshold, + ) } } // runBatch hands every name to the worker pool and waits for that batch to -// finish. It returns the bytes and the number of objects actually evicted. -func (q *lruQueue) runBatch(names []string) (int64, int32) { +// finish. +func (q *lruQueue) runBatch(names []string) { var wg sync.WaitGroup - var freed atomic.Int64 - var evicted atomic.Int32 wg.Add(len(names)) for i, name := range names { select { - case q.uploadChan <- uploadJob{name: name, wg: &wg, freed: &freed, evicted: &evicted}: + case q.uploadChan <- uploadJob{name: name, wg: &wg}: case <-q.doneChan: // shutting down - hand back everything we could not dispatch for _, undelivered := range names[i:] { @@ -388,12 +358,10 @@ func (q *lruQueue) runBatch(names []string) (int64, int32) { wg.Done() } wg.Wait() - return freed.Load(), evicted.Load() + return } } wg.Wait() - - return freed.Load(), evicted.Load() } // nominate detaches least-recently-used nodes until their combined size covers @@ -409,7 +377,6 @@ func (q *lruQueue) nominate(bytesNeeded int64, limit int) []string { q.mu.Lock() defer q.mu.Unlock() - now := time.Now() var names []string var busy []*lruNode var total int64 @@ -422,9 +389,6 @@ func (q *lruQueue) nominate(bytesNeeded int64, limit int) []string { case node.state != nodeQueued: // a worker owns it; it should not be in the list at all - case now.Before(node.retryAfter): - // backing off after a failed upload - case q.fileLocks.Get(node.name).Count() > 0: busy = append(busy, node) @@ -468,9 +432,8 @@ func (q *lruQueue) claimed(name string) bool { } // requeue returns a nominated node to the queue. A failed upload goes to the -// tail with a backoff, so one unwritable object cannot monopolise the workers. -// An object that was merely busy goes to the head, because being in use is what -// the head of an LRU means. +// tail for a future eviction cycle. An object that was merely busy goes to the +// head, because being in use is what the head of an LRU means. func (q *lruQueue) requeue(name string, failed bool) { q.mu.Lock() defer q.mu.Unlock() @@ -490,8 +453,6 @@ func (q *lruQueue) requeue(name string, failed bool) { node.state = nodeQueued if failed { - node.failures++ - node.retryAfter = time.Now().Add(retryBackoff(node.failures)) q.setTail(node) return } @@ -508,11 +469,6 @@ func (q *lruQueue) drop(name string) { } q.nodeMap.Delete(name) } - -func retryBackoff(failures uint32) time.Duration { - return min(time.Second*time.Duration(1< Date: Mon, 31 Aug 2026 17:00:07 -0600 Subject: [PATCH 50/89] Cache used space --- component/tiered_storage/cache_size.go | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 component/tiered_storage/cache_size.go diff --git a/component/tiered_storage/cache_size.go b/component/tiered_storage/cache_size.go new file mode 100644 index 000000000..8c0c7a5fc --- /dev/null +++ b/component/tiered_storage/cache_size.go @@ -0,0 +1,120 @@ +/* + Licensed under the MIT License . + + Copyright © 2026 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package tiered_storage + +import ( + "sync/atomic" + "time" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/log" +) + +// driftWarnBytes is how far the running total may diverge from a measurement +// before the correction is worth a log line. +const driftWarnBytes = 16 * common.MbToBytes + +// cacheSizeTracker tracks how many bytes of local storage the component is +// using. +// +// The total is maintained incrementally, because measuring it costs a du(1) +// subprocess on Linux and a full tree walk on Windows - far too expensive to do +// per write or per eviction tick. Incremental accounting drifts (directory +// entries, sparse files, anything that touches the cache directory behind our +// back), so Reconcile periodically replaces the total with a real measurement. +// +// Sizes are apparent bytes, matching `du -sb` on Linux. Windows measures +// allocated sectors instead, so the two platforms disagree slightly for small +// files; Reconcile is what keeps that from accumulating. +type cacheSizeTracker struct { + path string + used atomic.Int64 + reconcileInterval time.Duration + lastReconcile atomic.Int64 // unix nanoseconds +} + +func newCacheSizeTracker(path string, reconcileInterval time.Duration) *cacheSizeTracker { + t := &cacheSizeTracker{path: path, reconcileInterval: reconcileInterval} + t.lastReconcile.Store(time.Now().UnixNano()) + return t +} + +// Used returns the current cache usage in bytes. +func (t *cacheSizeTracker) Used() int64 { + return t.used.Load() +} + +// Add applies a signed change in bytes. The total is clamped at zero: drift +// could otherwise drive it negative and hide a full cache. +func (t *cacheSizeTracker) Add(delta int64) { + if delta == 0 { + return + } + for { + current := t.used.Load() + next := max(current+delta, 0) + if t.used.CompareAndSwap(current, next) { + return + } + } +} + +// Resize records one cached file changing from oldSize to newSize bytes. +func (t *cacheSizeTracker) Resize(oldSize, newSize int64) { + t.Add(newSize - oldSize) +} + +// Refresh measures the cache directory and replaces the running total. +// Changes made while the measurement is in flight are lost, which is inherent +// to reconciling against a moving target and is why it is not the hot path. +func (t *cacheSizeTracker) Refresh() { + t.lastReconcile.Store(time.Now().UnixNano()) + + usage, err := common.GetUsage(t.path) + if err != nil { + log.Err("cacheSizeTracker::Refresh : failed to measure %s [%v]", t.path, err) + return + } + + measured := int64(usage) + previous := t.used.Swap(measured) + if drift := measured - previous; drift > driftWarnBytes || drift < -driftWarnBytes { + log.Info( + "cacheSizeTracker::Refresh : corrected usage by %d bytes (was %d, now %d)", + drift, + previous, + measured, + ) + } +} + +// Reconcile refreshes the running total, but no more often than +// reconcileInterval. An interval of zero refreshes on every call. +func (t *cacheSizeTracker) Reconcile() { + if time.Since(time.Unix(0, t.lastReconcile.Load())) < t.reconcileInterval { + return + } + t.Refresh() +} From 727fe5321ff2f24dabeb1c5d724872d2b7e8b53a Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 17:27:09 -0600 Subject: [PATCH 51/89] Prevent path traversal exploits in local storage --- component/tiered_storage/tiered_storage.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 2c21fc31e..239e66d8d 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -33,6 +33,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "sync" "sync/atomic" "syscall" @@ -235,6 +236,21 @@ func (c *TieredStorage) Configure(_ bool) error { func (c *TieredStorage) OnConfigChange() { } +// localPath resolves an object name beneath the tiered storage root. +func (c *TieredStorage) localPath(name string) (string, error) { + name = filepath.FromSlash(common.NormalizeObjectName(name)) + if filepath.IsAbs(name) || filepath.VolumeName(name) != "" { + return "", syscall.EINVAL + } + + path := filepath.Join(c.tmpPath, name) + rel, err := filepath.Rel(c.tmpPath, path) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", syscall.EINVAL + } + return path, nil +} + // Directory operations func (c *TieredStorage) CreateDir(options internal.CreateDirOptions) error { return nil From 6776993aaf996179f5642e9bf290fa42a9a22b1c Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 19:24:55 -0600 Subject: [PATCH 52/89] Add local tier attribute lookup --- component/tiered_storage/tiered_storage.go | 40 ++++++++++++- .../tiered_storage/tiered_storage_linux.go | 56 ++++++++++++++++++ .../tiered_storage/tiered_storage_windows.go | 58 +++++++++++++++++++ 3 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 component/tiered_storage/tiered_storage_linux.go create mode 100644 component/tiered_storage/tiered_storage_windows.go diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 239e66d8d..4a47abb8c 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -942,7 +942,45 @@ func (c *TieredStorage) clearHandleDirty(handle *handlemap.Handle) { // Filesystem level operations func (c *TieredStorage) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { - return c.NextComponent().GetAttr(options) + localPath, err := c.localPath(options.Name) + if err != nil { + return nil, err + } + + flock := c.fileLocks.Get(options.Name) + flock.RLock() + defer flock.RUnlock() + return c.getAttrUnlocked(options, localPath) +} + +// getAttrUnlocked merges local and cloud attributes. The object's file lock +// must already be held. +func (c *TieredStorage) getAttrUnlocked( + options internal.GetAttrOptions, + localPath string, +) (*internal.ObjAttr, error) { + info, localErr := os.Stat(localPath) + if localErr != nil && !errors.Is(localErr, os.ErrNotExist) { + return nil, localErr + } + + attrs, cloudErr := c.NextComponent().GetAttr(options) + if localErr != nil { + return attrs, cloudErr + } + + localAttrs := newTieredStorageObjAttr(options.Name, info) + if cloudErr != nil || attrs == nil { + return localAttrs, nil + } + if info.IsDir() { + return attrs, nil + } + + merged := *attrs + merged.Size = localAttrs.Size + merged.Mtime = localAttrs.Mtime + return &merged, nil } func (c *TieredStorage) Chmod(options internal.ChmodOptions) error { diff --git a/component/tiered_storage/tiered_storage_linux.go b/component/tiered_storage/tiered_storage_linux.go new file mode 100644 index 000000000..d110825f0 --- /dev/null +++ b/component/tiered_storage/tiered_storage_linux.go @@ -0,0 +1,56 @@ +//go:build linux + +/* + Licensed under the MIT License . + + Copyright © 2026 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package tiered_storage + +import ( + "io/fs" + "os" + "syscall" + "time" + + "github.com/Seagate/cloudfuse/internal" +) + +func newTieredStorageObjAttr(path string, info fs.FileInfo) *internal.ObjAttr { + stat := info.Sys().(*syscall.Stat_t) + attrs := &internal.ObjAttr{ + Path: path, + Name: info.Name(), + Size: info.Size(), + Mode: info.Mode(), + Mtime: time.Unix(stat.Mtim.Sec, stat.Mtim.Nsec), + Atime: time.Unix(stat.Atim.Sec, stat.Atim.Nsec), + Ctime: time.Unix(stat.Ctim.Sec, stat.Ctim.Nsec), + } + + if info.Mode()&os.ModeSymlink != 0 { + attrs.Flags.Set(internal.PropFlagSymlink) + } else if info.IsDir() { + attrs.Flags.Set(internal.PropFlagIsDir) + } + return attrs +} diff --git a/component/tiered_storage/tiered_storage_windows.go b/component/tiered_storage/tiered_storage_windows.go new file mode 100644 index 000000000..b11b628d7 --- /dev/null +++ b/component/tiered_storage/tiered_storage_windows.go @@ -0,0 +1,58 @@ +//go:build windows + +/* + Licensed under the MIT License . + + Copyright © 2026 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package tiered_storage + +import ( + "io/fs" + "os" + "syscall" + "time" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/internal" +) + +func newTieredStorageObjAttr(path string, info fs.FileInfo) *internal.ObjAttr { + stat := info.Sys().(*syscall.Win32FileAttributeData) + attrs := &internal.ObjAttr{ + Path: common.NormalizeObjectName(path), + Name: common.NormalizeObjectName(info.Name()), + Size: info.Size(), + Mode: info.Mode() &^ os.ModePerm, + Mtime: time.Unix(0, stat.LastWriteTime.Nanoseconds()), + Atime: time.Unix(0, stat.LastAccessTime.Nanoseconds()), + Ctime: time.Unix(0, stat.CreationTime.Nanoseconds()), + } + + attrs.Flags.Set(internal.PropFlagModeDefault) + if info.Mode()&os.ModeSymlink != 0 { + attrs.Flags.Set(internal.PropFlagSymlink) + } else if info.IsDir() { + attrs.Flags.Set(internal.PropFlagIsDir) + } + return attrs +} From 299536c27b756fbdf0f76a895b0e2e3a834656d1 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 19:36:29 -0600 Subject: [PATCH 53/89] Persist tiered storage state --- component/tiered_storage/lru_policy.go | 2 - component/tiered_storage/persistence.go | 237 +++++++++++++++++++++ component/tiered_storage/tiered_storage.go | 14 +- 3 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 component/tiered_storage/persistence.go diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index facd8f405..698afbc3a 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -136,8 +136,6 @@ func (q *lruQueue) StartPolicy() error { q.maxEviction = defaultMaxEviction } - q.head = nil - q.tail = nil q.doneChan = make(chan struct{}) q.uploadChan = make(chan uploadJob, q.numWorkers*uploadQueueDepthPerWorker) diff --git a/component/tiered_storage/persistence.go b/component/tiered_storage/persistence.go new file mode 100644 index 000000000..d225128f9 --- /dev/null +++ b/component/tiered_storage/persistence.go @@ -0,0 +1,237 @@ +/* + Licensed under the MIT License . + + Copyright © 2026 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +*/ + +package tiered_storage + +import ( + "bytes" + "encoding/gob" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "slices" + + "github.com/Seagate/cloudfuse/common" +) + +const ( + tieredStorageSnapshotPath = ".tieredStorageSnapshot.gob" + tieredStorageSnapshotVersion = 1 +) + +type persistedFileState struct { + Size int64 + Mtime int64 + CloudBacked bool + Dirty bool +} + +type tieredStorageSnapshot struct { + Version uint32 + Files map[string]persistedFileState + LRUOrder []string +} + +type recoveredFile struct { + name string + info fs.FileInfo + state persistedFileState +} + +func (c *TieredStorage) writeSnapshot() error { + snapshot := tieredStorageSnapshot{ + Version: tieredStorageSnapshotVersion, + Files: make(map[string]persistedFileState), + } + + c.fileMap.Range(func(key, value any) bool { + name := key.(string) + localPath, err := c.localPath(name) + if err != nil { + return true + } + info, err := os.Stat(localPath) + if err != nil || !info.Mode().IsRegular() { + return true + } + node := value.(*FileNode) + snapshot.Files[name] = persistedFileState{ + Size: info.Size(), + Mtime: info.ModTime().UnixNano(), + CloudBacked: node.cloudBacked.Load(), + Dirty: node.isDirty.Load(), + } + return true + }) + + c.policy.mu.Lock() + for node := c.policy.head; node != nil; node = node.next { + snapshot.LRUOrder = append(snapshot.LRUOrder, node.name) + } + c.policy.mu.Unlock() + + var data bytes.Buffer + if err := gob.NewEncoder(&data).Encode(snapshot); err != nil { + return fmt.Errorf("encode state snapshot: %w", err) + } + + path := filepath.Join(c.tmpPath, tieredStorageSnapshotPath) + tmpPath := path + ".tmp" + file, err := common.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("create state snapshot: %w", err) + } + if _, err = file.Write(data.Bytes()); err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("write state snapshot: %w", err) + } + if err := replaceFile(tmpPath, path); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("install state snapshot: %w", err) + } + return nil +} + +func (c *TieredStorage) readSnapshot() (*tieredStorageSnapshot, error) { + path := filepath.Join(c.tmpPath, tieredStorageSnapshotPath) + _ = os.Remove(path + ".tmp") + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + defer os.Remove(path) + + var snapshot tieredStorageSnapshot + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&snapshot); err != nil { + return nil, fmt.Errorf("decode state snapshot: %w", err) + } + if snapshot.Version != tieredStorageSnapshotVersion { + return nil, fmt.Errorf("unsupported state snapshot version %d", snapshot.Version) + } + return &snapshot, nil +} + +func (c *TieredStorage) recoverLocalState(snapshot *tieredStorageSnapshot) error { + var recovered []recoveredFile + err := filepath.WalkDir(c.tmpPath, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || !entry.Type().IsRegular() { + return nil + } + if entry.Name() == tieredStorageSnapshotPath || + entry.Name() == tieredStorageSnapshotPath+".tmp" { + return nil + } + + rel, err := filepath.Rel(c.tmpPath, path) + if err != nil { + return err + } + name := common.NormalizeObjectName(filepath.ToSlash(rel)) + info, err := entry.Info() + if err != nil { + return err + } + + state := persistedFileState{ + Size: info.Size(), + Mtime: info.ModTime().UnixNano(), + Dirty: true, + } + if snapshot != nil { + if saved, found := snapshot.Files[name]; found && + saved.Size == info.Size() && saved.Mtime == info.ModTime().UnixNano() { + state = saved + } + } + + if state.CloudBacked && !state.Dirty { + if err := os.Remove(path); err != nil { + return err + } + c.cacheSize.Add(-info.Size()) + return nil + } + if !state.CloudBacked { + state.Dirty = true + } + recovered = append(recovered, recoveredFile{name: name, info: info, state: state}) + return nil + }) + if err != nil { + return err + } + + byName := make(map[string]recoveredFile, len(recovered)) + for _, file := range recovered { + byName[file.name] = file + node := &FileNode{name: file.name} + node.size.Store(file.info.Size()) + node.cloudBacked.Store(file.state.CloudBacked) + node.isDirty.Store(file.state.Dirty) + c.fileMap.Store(file.name, node) + } + + var order []string + queued := make(map[string]struct{}, len(recovered)) + if snapshot != nil { + for _, name := range snapshot.LRUOrder { + if _, found := byName[name]; found { + order = append(order, name) + queued[name] = struct{}{} + } + } + } + + var unmatched []recoveredFile + for _, file := range recovered { + if _, found := queued[file.name]; !found { + unmatched = append(unmatched, file) + } + } + slices.SortFunc(unmatched, func(a, b recoveredFile) int { + return b.info.ModTime().Compare(a.info.ModTime()) + }) + for _, file := range unmatched { + order = append(order, file.name) + } + + for _, name := range slices.Backward(order) { + c.policy.Enqueue(name) + } + return nil +} diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 4a47abb8c..adbc1912c 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -136,10 +136,17 @@ func (c *TieredStorage) Start(ctx context.Context) error { // A crash can leave partial downloads behind. They are not valid object // data, so remove them before anything can list, open or upload them. c.removePartialDownloads() + snapshot, err := c.readSnapshot() + if err != nil { + log.Warn("TieredStorage::Start : ignoring invalid state snapshot [%v]", err) + } // Seed the usage counter from what is actually on disk. This is the one // place where measuring the whole directory is worth its cost. c.cacheSize.Refresh() + if err := c.recoverLocalState(snapshot); err != nil { + return fmt.Errorf("TieredStorage: failed to recover local state: %w", err) + } if c.policy != nil { if err := c.policy.StartPolicy(); err != nil { @@ -159,10 +166,11 @@ func (c *TieredStorage) Stop() error { log.Trace("TieredStorage::Stop : Stopping component %s", c.Name()) if c.policy != nil { - return c.policy.StopPolicy() + if err := c.policy.StopPolicy(); err != nil { + return err + } } - - return nil + return c.writeSnapshot() } // removePartialDownloads deletes interrupted downloads left by a previous run. From ee89ba2608d416255fe7193cac24db767c8be8d0 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 19:44:02 -0600 Subject: [PATCH 54/89] Honor file creation and open semantics --- component/tiered_storage/tiered_storage.go | 187 +++++++++++++-------- 1 file changed, 120 insertions(+), 67 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index adbc1912c..560d6bf06 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -33,6 +33,8 @@ import ( "io/fs" "os" "path/filepath" + "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -298,8 +300,11 @@ func (c *TieredStorage) createFileUnlocked( // WriteFile reserves space as the file grows. //Create the file in the local cache, we will ignore the create empty and cloud stuff for now - localPath := filepath.Join(c.tmpPath, options.Name) - err := os.MkdirAll(filepath.Dir(localPath), 0755) + localPath, err := c.localPath(options.Name) + if err != nil { + return nil, err + } + err = os.MkdirAll(filepath.Dir(localPath), 0755) if err != nil { return nil, err @@ -308,8 +313,8 @@ func (c *TieredStorage) createFileUnlocked( //Open local file localFile, err := common.OpenFile( localPath, - os.O_CREATE|os.O_RDWR, - options.Mode, + os.O_CREATE|os.O_EXCL|os.O_RDWR, + c.cacheFileMode(options.Mode), ) if err != nil { return nil, err @@ -337,6 +342,21 @@ func (c *TieredStorage) CreateFile( flock := c.fileLocks.Get(options.Name) flock.Lock() defer flock.Unlock() + + localPath, err := c.localPath(options.Name) + if err != nil { + return nil, err + } + _, err = c.getAttrUnlocked( + internal.GetAttrOptions{Name: options.Name}, + localPath, + ) + if err == nil { + return nil, syscall.EEXIST + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + handle, err := c.createFileUnlocked(options) if err != nil { return nil, err @@ -382,79 +402,83 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H flock.Lock() defer flock.Unlock() - //Go through flag cases, might need to explore O_TRUNC - - //Case 1: OpenFile with O_Create - if options.Flags&os.O_CREATE != 0 { - //Check if file first exists, then proceed - _, exists := c.fileMap.Load(options.Name) - if !exists { - handle, err := c.createFileUnlocked( - internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, - ) - if err != nil { - return nil, err - } - flock.Inc() - return handle, nil + localPath, err := c.localPath(options.Name) + if err != nil { + return nil, err + } + attrs, attrErr := c.getAttrUnlocked(internal.GetAttrOptions{Name: options.Name}, localPath) + if attrErr != nil && !errors.Is(attrErr, os.ErrNotExist) { + return nil, attrErr + } + exists := attrErr == nil + if options.Flags&os.O_CREATE != 0 && options.Flags&os.O_EXCL != 0 && exists { + return nil, syscall.EEXIST + } + if !exists { + if options.Flags&os.O_CREATE == 0 { + return nil, syscall.ENOENT + } + handle, err := c.createFileUnlocked( + internal.CreateFileOptions{Name: options.Name, Mode: options.Mode}, + ) + if err != nil { + return nil, err } + flock.Inc() + return handle, nil } - //1. Initial Check Map - _, exists := c.fileMap.Load(options.Name) - - //if exists skip to opening file since it should already be in local cache - if !exists { - //2. Check if File exists in Disk, if not check cloud - info, err := os.Stat(filepath.Join(c.tmpPath, options.Name)) - if err == nil { + _, tracked := c.fileMap.Load(options.Name) + if !tracked { + info, statErr := os.Stat(localPath) + if statErr == nil { log.Warn( "TieredStorage::OpenFile : Warning file exists locally on disk but not in tiered storage cache: %s", options.Name, ) - //Read from local disk, create file node and add to file map node := &FileNode{name: options.Name} node.size.Store(info.Size()) + node.isDirty.Store(true) c.fileMap.Store(options.Name, node) - } else { - //3. Check if File exists in Cloud - info, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}) - if err != nil { - // file does not exist in cloud, return error - log.Err("TieredStorage::OpenFile : File Does not exist in cloud or local cache: %s", - options.Name, - ) - return nil, err - } - // file exists in cloud, create local copy (name doesn't matter)and add to file map localCopyNode := &FileNode{name: options.Name} - localCopyNode.size.Store(info.Size) + localCopyNode.size.Store(attrs.Size) localCopyNode.cloudBacked.Store(true) - // make room for the copy we are about to download - if err := c.reserveSpace(info.Size); err != nil { - return nil, err - } - //download it to the local cache and add to file map - err = c.downloadCopyFromCloud(options) - if err != nil { - return nil, err + if options.Flags&os.O_TRUNC != 0 { + if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { + return nil, err + } + file, err := common.OpenFile( + localPath, + os.O_CREATE|os.O_EXCL|os.O_RDWR, + c.cacheFileMode(options.Mode), + ) + if err != nil { + return nil, err + } + if err := file.Close(); err != nil { + return nil, err + } + localCopyNode.size.Store(0) + localCopyNode.isDirty.Store(true) + } else { + if err := c.reserveSpace(attrs.Size); err != nil { + return nil, err + } + if err := c.downloadCopyFromCloud(options); err != nil { + return nil, err + } } c.fileMap.Store(options.Name, localCopyNode) - } - } - //At this point the file should be in the local cache, so we can proceed to open it - - //Open the file in the local cache - localPath := filepath.Join(c.tmpPath, options.Name) + openFlags := options.Flags &^ (os.O_CREATE | os.O_EXCL) localFile, err := common.OpenFile( localPath, - os.O_RDWR, - options.Mode, + openFlags, + c.cacheFileMode(options.Mode), ) if err != nil { return nil, err @@ -466,6 +490,14 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H if options.Flags&os.O_APPEND != 0 { handle.Flags.Set(handlemap.HandleOpenedAppend) } + if options.Flags&os.O_TRUNC != 0 { + if value, found := c.fileMap.Load(options.Name); found { + node := value.(*FileNode) + c.recordSize(node, 0) + node.isDirty.Store(true) + } + c.setHandleDirty(handle) + } //increase handle count flock.Inc() @@ -473,6 +505,13 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H return handle, nil } +func (c *TieredStorage) cacheFileMode(mode os.FileMode) os.FileMode { + if mode == 0 || runtime.GOOS == "windows" { + return common.DefaultFilePermissionBits + } + return mode +} + // downloadCopyFromCloud caches a cloud object on local storage. // // The data lands in a temporary file and is renamed into place only once it is @@ -480,8 +519,11 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H // real path, which matters because local data is authoritative here: a partial // download found at a real path would later be uploaded over the good cloud copy. func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) error { - localPath := filepath.Join(c.tmpPath, options.Name) - err := os.MkdirAll(filepath.Dir(localPath), 0755) + localPath, err := c.localPath(options.Name) + if err != nil { + return err + } + err = os.MkdirAll(filepath.Dir(localPath), 0755) if err != nil { return err } @@ -490,7 +532,7 @@ func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) partFile, err := common.OpenFile( partPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, - options.Mode, + c.cacheFileMode(options.Mode), ) if err != nil { return err @@ -545,10 +587,13 @@ func replaceFile(src, dst string) error { // purgeLocal removes an object's local copy and updates the usage counter. // The object's file lock must be held. func (c *TieredStorage) purgeLocal(name string) error { - localPath := filepath.Join(c.tmpPath, name) + localPath, err := c.localPath(name) + if err != nil { + return err + } info, statErr := os.Stat(localPath) - err := os.Remove(localPath) + err = os.Remove(localPath) if err == nil && statErr == nil { c.cacheSize.Add(-info.Size()) } @@ -772,8 +817,11 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { func (c *TieredStorage) uploadCachedFile(name string) error { //get the local path - localPath := filepath.Join(c.tmpPath, name) - _, err := os.Stat(localPath) + localPath, err := c.localPath(name) + if err != nil { + return err + } + _, err = os.Stat(localPath) if err != nil { log.Err("TieredStorage::uploadFile : %s stat failed [%v]", name, err) return err @@ -848,10 +896,15 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { } //Local only State, this will happen anyways if it exists local //Rename - err := os.Rename( - filepath.Join(c.tmpPath, options.Src), - filepath.Join(c.tmpPath, options.Dst), - ) + srcPath, err := c.localPath(options.Src) + if err != nil { + return err + } + dstPath, err := c.localPath(options.Dst) + if err != nil { + return err + } + err = os.Rename(srcPath, dstPath) if err != nil { return err } From 7b4bfc1fe5e9f92b5cf78c7b9dbc24e2ccd3756e Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 19:49:54 -0600 Subject: [PATCH 55/89] Merge local and cloud directory state --- component/tiered_storage/tiered_storage.go | 131 ++++++++++++++++++++- 1 file changed, 126 insertions(+), 5 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 560d6bf06..49ebae4d4 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -263,29 +263,144 @@ func (c *TieredStorage) localPath(name string) (string, error) { // Directory operations func (c *TieredStorage) CreateDir(options internal.CreateDirOptions) error { + if _, err := c.GetAttr(internal.GetAttrOptions{Name: options.Name}); err == nil { + return syscall.EEXIST + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + localPath, err := c.localPath(options.Name) + if err != nil { + return err + } + mode := options.Mode + if mode == 0 || runtime.GOOS == "windows" { + mode = common.DefaultDirectoryPermissionBits + } + if err := os.Mkdir(localPath, mode); err != nil { + return err + } + if err := c.NextComponent().CreateDir(options); err != nil && !errors.Is(err, os.ErrExist) { + return err + } return nil } func (c *TieredStorage) DeleteDir(options internal.DeleteDirOptions) error { + localPath, err := c.localPath(options.Name) + if err != nil { + return err + } + cloudErr := c.NextComponent().DeleteDir(options) + if cloudErr != nil && !errors.Is(cloudErr, os.ErrNotExist) { + return cloudErr + } + localErr := os.Remove(localPath) + if localErr != nil && !errors.Is(localErr, os.ErrNotExist) { + return localErr + } + if errors.Is(localErr, os.ErrNotExist) && errors.Is(cloudErr, os.ErrNotExist) { + return syscall.ENOENT + } return nil } func (c *TieredStorage) IsDirEmpty(options internal.IsDirEmptyOptions) bool { - return false + localPath, err := c.localPath(options.Name) + if err != nil { + return false + } + entries, localErr := os.ReadDir(localPath) + if localErr == nil && len(entries) > 0 { + return false + } + if localErr != nil && !errors.Is(localErr, os.ErrNotExist) { + return false + } + return c.NextComponent().IsDirEmpty(options) } func (c *TieredStorage) OpenDir(options internal.OpenDirOptions) error { - return nil + localPath, err := c.localPath(options.Name) + if err != nil { + return err + } + if info, err := os.Stat(localPath); err == nil && info.IsDir() { + return nil + } + return c.NextComponent().OpenDir(options) } func (c *TieredStorage) StreamDir( options internal.StreamDirOptions, ) ([]*internal.ObjAttr, string, error) { - return nil, "", nil + localPath, pathErr := c.localPath(options.Name) + if pathErr != nil { + return nil, "", pathErr + } + + attrs, token, cloudErr := c.NextComponent().StreamDir(options) + entries, localErr := os.ReadDir(localPath) + localExists := localErr == nil + if localErr != nil && !errors.Is(localErr, os.ErrNotExist) { + return nil, "", localErr + } + if cloudErr != nil && !(localExists && errors.Is(cloudErr, os.ErrNotExist)) { + return attrs, token, cloudErr + } + if cloudErr != nil { + attrs = nil + token = "" + } + + localAttrs := make(map[string]*internal.ObjAttr, len(entries)) + for _, entry := range entries { + if c.internalCacheEntry(options.Name, entry.Name()) { + continue + } + entryPath := common.JoinUnixFilepath(options.Name, entry.Name()) + info, err := entry.Info() + if err != nil { + return nil, "", err + } + localAttrs[entry.Name()] = newTieredStorageObjAttr(entryPath, info) + } + + listed := make(map[string]struct{}, len(attrs)) + for index, attr := range attrs { + listed[attr.Name] = struct{}{} + local, found := localAttrs[attr.Name] + if !found || attr.IsDir() { + continue + } + merged := *attr + merged.Size = local.Size + merged.Mtime = local.Mtime + attrs[index] = &merged + } + + if token == "" { + for name, attr := range localAttrs { + if _, found := listed[name]; !found { + attrs = append(attrs, attr) + } + } + } + slices.SortFunc(attrs, func(a, b *internal.ObjAttr) int { + return strings.Compare(a.Path, b.Path) + }) + return attrs, token, nil } func (c *TieredStorage) CloseDir(options internal.CloseDirOptions) error { - return nil + localPath, err := c.localPath(options.Name) + if err != nil { + return err + } + if info, err := os.Stat(localPath); err == nil && info.IsDir() { + return nil + } + return c.NextComponent().CloseDir(options) } func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { @@ -963,7 +1078,13 @@ func (c *TieredStorage) renameOpenHandles( } func (c *TieredStorage) SyncDir(options internal.SyncDirOptions) error { - return nil + return c.NextComponent().SyncDir(options) +} + +func (c *TieredStorage) internalCacheEntry(directory, name string) bool { + return directory == "" && + (name == tieredStorageSnapshotPath || name == tieredStorageSnapshotPath+".tmp") || + strings.HasSuffix(name, partialDownloadSuffix) } // Symlink operations From b883ff8b32bffd055f19df0d734aefe425750829 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 19:55:24 -0600 Subject: [PATCH 56/89] Normalize missing file delete errors --- component/tiered_storage/tiered_storage.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 49ebae4d4..906249934 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -492,7 +492,11 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { val, exists := c.fileMap.Load(options.Name) if !exists { // cloud only - return c.NextComponent().DeleteFile(options) + err := c.NextComponent().DeleteFile(options) + if errors.Is(err, os.ErrNotExist) { + return syscall.ENOENT + } + return err } node := val.(*FileNode) From 0a24d4a74211f357d73a7b73e46e3412a599309e Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 20:01:22 -0600 Subject: [PATCH 57/89] Expand tiered storage coverage --- component/tiered_storage/lru_policy_test.go | 40 +- .../tiered_storage/tiered_storage_test.go | 341 ++++++++++++++++++ 2 files changed, 355 insertions(+), 26 deletions(-) diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index 68da58345..36d22e4e7 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -268,12 +268,13 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { suite.assert.True(ex3) suite.assert.True(ex4) - //file4 should be in upload channel - time.Sleep(10 * time.Millisecond) - + suite.assert.Eventually(func() bool { + mu.Lock() + defer mu.Unlock() + return len(uploaded) >= 2 + }, time.Second, time.Millisecond) mu.Lock() - snapshot := make([]string, len(uploaded)) - copy(snapshot, uploaded) + snapshot := append([]string(nil), uploaded...) mu.Unlock() // file1 and file2 are the LRU tail so they should be evicted to reach targetRatio (60%) @@ -283,19 +284,6 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { suite.assert.NotContains(snapshot, "file3") suite.assert.NotContains(snapshot, "file4") - fmt.Println("=== nodeMap contents ===") - suite.policy.nodeMap.Range(func(key, value interface{}) bool { - lruNode := value.(*lruNode) - fmt.Printf(" key=%q, name=%q, next=%v, prev=%v\n", - key, - lruNode.name, - lruNode.next, - lruNode.prev, - ) - return true // continue iteration - }) - fmt.Println("=== end nodeMap ===") - _, ex1 = suite.policy.nodeMap.Load("file1") _, ex2 = suite.policy.nodeMap.Load("file2") _, ex3 = suite.policy.nodeMap.Load("file3") @@ -350,20 +338,20 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { suite.assert.NoError(err) suite.policy.Enqueue("file4") - time.Sleep(10 * time.Millisecond) - + suite.assert.Eventually(func() bool { + mu.Lock() + defer mu.Unlock() + return len(uploaded) >= 2 + }, time.Second, time.Millisecond) mu.Lock() - snapshot := make([]string, len(uploaded)) - copy(snapshot, uploaded) + snapshot := append([]string(nil), uploaded...) mu.Unlock() - fmt.Print(snapshot) - fmt.Print(suite.policy.head.name) - fmt.Print(suite.policy.tail.name) - //file1 should be head, file4 should be tail + suite.policy.mu.Lock() suite.assert.Equal("file1", suite.policy.head.name) suite.assert.Equal("file4", suite.policy.tail.name) + suite.policy.mu.Unlock() // file1 and file2 are the LRU tail so they should be evicted to reach targetRatio (60%) suite.assert.Contains(snapshot, "file2") diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index f5a1eb0cc..3fc819e2a 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -166,6 +166,347 @@ func (suite *tieredStorageTestSuite) cleanupTest() { suite.assert.NoError(err) } +func TestLocalPath(t *testing.T) { + storage := &TieredStorage{tmpPath: filepath.Join(string(os.PathSeparator), "cache")} + + tests := []struct { + name string + expected string + valid bool + }{ + {name: "", expected: storage.tmpPath, valid: true}, + {name: "dir/file", expected: filepath.Join(storage.tmpPath, "dir", "file"), valid: true}, + {name: `dir\file`, expected: filepath.Join(storage.tmpPath, "dir", "file"), valid: true}, + {name: "../file", valid: false}, + {name: "dir/../../file", valid: false}, + {name: filepath.Join(string(os.PathSeparator), "file"), valid: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + path, err := storage.localPath(test.name) + if test.valid { + assert.NoError(t, err) + assert.Equal(t, test.expected, path) + return + } + assert.ErrorIs(t, err, syscall.EINVAL) + }) + } +} + +func (suite *tieredStorageTestSuite) TestGetAttrLocalOnly() { + defer suite.cleanupTest() + + const path = "local-only" + handle, err := suite.tieredStorage.CreateFile( + internal.CreateFileOptions{Name: path, Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Data: []byte("local data")}, + ) + suite.Require().NoError(err) + + attrs, err := suite.tieredStorage.GetAttr(internal.GetAttrOptions{Name: path}) + suite.Require().NoError(err) + suite.assert.Equal(path, attrs.Path) + suite.assert.EqualValues(len("local data"), attrs.Size) +} + +func (suite *tieredStorageTestSuite) TestGetAttrCloudOnly() { + defer suite.cleanupTest() + + const path = "cloud-only" + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.Require().NoError(err) + _, err = suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Data: []byte("cloud data")}, + ) + suite.Require().NoError(err) + suite.Require().NoError(suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle})) + + attrs, err := suite.tieredStorage.GetAttr(internal.GetAttrOptions{Name: path}) + suite.Require().NoError(err) + suite.assert.EqualValues(len("cloud data"), attrs.Size) + suite.assert.NoFileExists(filepath.Join(suite.cache_path, path)) +} + +func (suite *tieredStorageTestSuite) TestGetAttrPrefersLocalData() { + defer suite.cleanupTest() + + const path = "local-and-cloud" + cloudHandle, err := suite.loopback.CreateFile( + internal.CreateFileOptions{Name: path, Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: cloudHandle, Data: []byte("cloud")}, + ) + suite.Require().NoError(err) + suite.Require().NoError( + suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: cloudHandle}), + ) + + handle, err := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: path, Flags: os.O_RDWR, Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Offset: 0, Data: []byte("local data")}, + ) + suite.Require().NoError(err) + + attrs, err := suite.tieredStorage.GetAttr(internal.GetAttrOptions{Name: path}) + suite.Require().NoError(err) + suite.assert.EqualValues(len("local data"), attrs.Size) + + cloudAttrs, err := suite.loopback.GetAttr(internal.GetAttrOptions{Name: path}) + suite.Require().NoError(err) + suite.assert.EqualValues(len("cloud"), cloudAttrs.Size) +} + +func (suite *tieredStorageTestSuite) TestRestartRestoresLocalStateAndLRUOrder() { + defer suite.cleanupTest() + + for _, path := range []string{"older", "newer"} { + handle, err := suite.tieredStorage.CreateFile( + internal.CreateFileOptions{Name: path, Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: handle, Data: []byte(path)}, + ) + suite.Require().NoError(err) + suite.Require().NoError( + suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}), + ) + } + + suite.Require().NoError(suite.tieredStorage.Stop()) + restarted := newTestTieredStorage(suite.loopback) + suite.Require().NoError(restarted.Start(context.Background())) + suite.tieredStorage = restarted + + for _, path := range []string{"older", "newer"} { + value, found := restarted.fileMap.Load(path) + suite.Require().True(found) + node := value.(*FileNode) + suite.assert.False(node.cloudBacked.Load()) + suite.assert.True(node.isDirty.Load()) + _, queued := restarted.policy.nodeMap.Load(path) + suite.assert.True(queued) + } + suite.Require().NotNil(restarted.policy.head) + suite.Require().NotNil(restarted.policy.tail) + suite.assert.Equal("newer", restarted.policy.head.name) + suite.assert.Equal("older", restarted.policy.tail.name) + suite.assert.NoFileExists(filepath.Join(suite.cache_path, tieredStorageSnapshotPath)) +} + +func (suite *tieredStorageTestSuite) TestRestartWithoutSnapshotRecoversConservatively() { + defer suite.cleanupTest() + + suite.Require().NoError(suite.tieredStorage.Stop()) + suite.Require().NoError(os.Remove(filepath.Join(suite.cache_path, tieredStorageSnapshotPath))) + suite.Require().NoError( + os.WriteFile(filepath.Join(suite.cache_path, "recovered"), []byte("data"), 0644), + ) + + restarted := newTestTieredStorage(suite.loopback) + suite.Require().NoError(restarted.Start(context.Background())) + suite.tieredStorage = restarted + + value, found := restarted.fileMap.Load("recovered") + suite.Require().True(found) + node := value.(*FileNode) + suite.assert.False(node.cloudBacked.Load()) + suite.assert.True(node.isDirty.Load()) + _, queued := restarted.policy.nodeMap.Load("recovered") + suite.assert.True(queued) +} + +func (suite *tieredStorageTestSuite) TestCreateFileRejectsExistingCloudObject() { + defer suite.cleanupTest() + + const path = "existing-cloud" + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.Require().NoError(err) + suite.Require().NoError(suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle})) + + _, err = suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.assert.ErrorIs(err, syscall.EEXIST) +} + +func (suite *tieredStorageTestSuite) TestCreateFileRejectsExistingLocalObject() { + defer suite.cleanupTest() + + const path = "existing-local" + _, err := suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.Require().NoError(err) + + _, err = suite.tieredStorage.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.assert.ErrorIs(err, syscall.EEXIST) +} + +func (suite *tieredStorageTestSuite) TestOpenFileExclusiveCreateRejectsExistingCloudObject() { + defer suite.cleanupTest() + + const path = "exclusive-cloud" + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.Require().NoError(err) + suite.Require().NoError(suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle})) + + _, err = suite.tieredStorage.OpenFile(internal.OpenFileOptions{ + Name: path, Flags: os.O_CREATE | os.O_EXCL | os.O_RDWR, Mode: 0644, + }) + suite.assert.ErrorIs(err, syscall.EEXIST) +} + +func (suite *tieredStorageTestSuite) TestOpenFileTruncatesCloudObjectWithoutDownloading() { + defer suite.cleanupTest() + + const path = "truncate-cloud" + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.Require().NoError(err) + _, err = suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Data: []byte("old cloud data")}, + ) + suite.Require().NoError(err) + suite.Require().NoError(suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle})) + + handle, err = suite.tieredStorage.OpenFile(internal.OpenFileOptions{ + Name: path, Flags: os.O_WRONLY | os.O_TRUNC, Mode: 0644, + }) + suite.Require().NoError(err) + suite.assert.True(handle.Dirty()) + info, err := os.Stat(filepath.Join(suite.cache_path, path)) + suite.Require().NoError(err) + suite.assert.Zero(info.Size()) + suite.Require().NoError( + suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}), + ) + + attrs, err := suite.loopback.GetAttr(internal.GetAttrOptions{Name: path}) + suite.Require().NoError(err) + suite.assert.Zero(attrs.Size) +} + +func (suite *tieredStorageTestSuite) TestOpenFileHonorsReadOnlyFlag() { + defer suite.cleanupTest() + + const path = "read-only" + data := []byte("cloud data") + handle, err := suite.loopback.CreateFile(internal.CreateFileOptions{Name: path, Mode: 0644}) + suite.Require().NoError(err) + _, err = suite.loopback.WriteFile(&internal.WriteFileOptions{Handle: handle, Data: data}) + suite.Require().NoError(err) + suite.Require().NoError(suite.loopback.ReleaseFile(internal.ReleaseFileOptions{Handle: handle})) + + handle, err = suite.tieredStorage.OpenFile(internal.OpenFileOptions{ + Name: path, Flags: os.O_RDONLY, + }) + suite.Require().NoError(err) + buffer := make([]byte, len(data)) + _, err = suite.tieredStorage.ReadInBuffer( + &internal.ReadInBufferOptions{Handle: handle, Data: buffer}, + ) + suite.Require().NoError(err) + suite.assert.Equal(data, buffer) + suite.Require().NoError( + suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}), + ) +} + +func (suite *tieredStorageTestSuite) TestStreamDirMergesLocalAndCloudEntries() { + defer suite.cleanupTest() + + localHandle, err := suite.tieredStorage.CreateFile( + internal.CreateFileOptions{Name: "local", Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: localHandle, Data: []byte("local data")}, + ) + suite.Require().NoError(err) + + for _, path := range []string{"cloud", "shared"} { + handle, err := suite.loopback.CreateFile( + internal.CreateFileOptions{Name: path, Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.loopback.WriteFile( + &internal.WriteFileOptions{Handle: handle, Data: []byte("cloud")}, + ) + suite.Require().NoError(err) + suite.Require().NoError( + suite.loopback.ReleaseFile( + internal.ReleaseFileOptions{Handle: handle}, + ), + ) + } + sharedHandle, err := suite.tieredStorage.OpenFile( + internal.OpenFileOptions{Name: "shared", Flags: os.O_RDWR, Mode: 0644}, + ) + suite.Require().NoError(err) + _, err = suite.tieredStorage.WriteFile( + &internal.WriteFileOptions{Handle: sharedHandle, Data: []byte("local data")}, + ) + suite.Require().NoError(err) + + attrs, token, err := suite.tieredStorage.StreamDir(internal.StreamDirOptions{}) + suite.Require().NoError(err) + suite.assert.Empty(token) + suite.Require().Len(attrs, 3) + suite.assert.Equal([]string{"cloud", "local", "shared"}, []string{ + attrs[0].Name, attrs[1].Name, attrs[2].Name, + }) + suite.assert.EqualValues(len("local data"), attrs[1].Size) + suite.assert.EqualValues(len("local data"), attrs[2].Size) +} + +func (suite *tieredStorageTestSuite) TestCreateAndDeleteLocalDirectory() { + defer suite.cleanupTest() + + const path = "directory" + suite.Require().NoError( + suite.tieredStorage.CreateDir(internal.CreateDirOptions{Name: path, Mode: 0755}), + ) + attrs, err := suite.tieredStorage.GetAttr(internal.GetAttrOptions{Name: path}) + suite.Require().NoError(err) + suite.assert.True(attrs.IsDir()) + suite.assert.True(suite.tieredStorage.IsDirEmpty(internal.IsDirEmptyOptions{Name: path})) + suite.Require().NoError(suite.tieredStorage.DeleteDir(internal.DeleteDirOptions{Name: path})) + _, err = suite.tieredStorage.GetAttr(internal.GetAttrOptions{Name: path}) + suite.assert.ErrorIs(err, syscall.ENOENT) +} + +func (suite *tieredStorageTestSuite) TestStreamDirListsImplicitLocalDirectory() { + defer suite.cleanupTest() + + handle, err := suite.tieredStorage.CreateFile( + internal.CreateFileOptions{Name: "directory/file", Mode: 0644}, + ) + suite.Require().NoError(err) + suite.Require().NoError( + suite.tieredStorage.ReleaseFile(internal.ReleaseFileOptions{Handle: handle}), + ) + suite.Require().NoError( + suite.tieredStorage.OpenDir(internal.OpenDirOptions{Name: "directory"}), + ) + + attrs, token, err := suite.tieredStorage.StreamDir( + internal.StreamDirOptions{Name: "directory"}, + ) + suite.Require().NoError(err) + suite.assert.Empty(token) + suite.Require().Len(attrs, 1) + suite.assert.Equal("directory/file", attrs[0].Path) + suite.Require().NoError( + suite.tieredStorage.CloseDir(internal.CloseDirOptions{Name: "directory"}), + ) +} + //Testing OpenFile func (suite *tieredStorageTestSuite) TestOpenFileNotInCache() { From fe54f7ee6b39982b530a48feb640d08a1bf92b4f Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 21:53:35 -0600 Subject: [PATCH 58/89] Don't buffer the upload channel --- component/tiered_storage/lru_policy.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 698afbc3a..c088e5d53 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -35,12 +35,6 @@ import ( "github.com/Seagate/cloudfuse/common/log" ) -const ( - // uploadQueueDepthPerWorker sizes the job channel. It only needs enough - // slack that the dispatcher is not serialised against the workers. - uploadQueueDepthPerWorker = 16 -) - // nodeState records whether the queue or an eviction worker owns a node. type nodeState uint8 @@ -137,7 +131,7 @@ func (q *lruQueue) StartPolicy() error { } q.doneChan = make(chan struct{}) - q.uploadChan = make(chan uploadJob, q.numWorkers*uploadQueueDepthPerWorker) + q.uploadChan = make(chan uploadJob) q.workerWg.Add(q.numWorkers) for range q.numWorkers { From 2d76d42a29dbe259d2258de88c36fd9820fc6747 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 21:54:30 -0600 Subject: [PATCH 59/89] Remove resize (trivial wrapper for Add) --- component/tiered_storage/cache_size.go | 8 +------- component/tiered_storage/tiered_storage.go | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/component/tiered_storage/cache_size.go b/component/tiered_storage/cache_size.go index 8c0c7a5fc..667091bad 100644 --- a/component/tiered_storage/cache_size.go +++ b/component/tiered_storage/cache_size.go @@ -81,11 +81,6 @@ func (t *cacheSizeTracker) Add(delta int64) { } } -// Resize records one cached file changing from oldSize to newSize bytes. -func (t *cacheSizeTracker) Resize(oldSize, newSize int64) { - t.Add(newSize - oldSize) -} - // Refresh measures the cache directory and replaces the running total. // Changes made while the measurement is in flight are lost, which is inherent // to reconciling against a moving target and is why it is not the hot path. @@ -110,8 +105,7 @@ func (t *cacheSizeTracker) Refresh() { } } -// Reconcile refreshes the running total, but no more often than -// reconcileInterval. An interval of zero refreshes on every call. +// Reconcile refreshes the running total, but no more often than reconcileInterval. func (t *cacheSizeTracker) Reconcile() { if time.Since(time.Unix(0, t.lastReconcile.Load())) < t.reconcileInterval { return diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 906249934..0b7ea1279 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -725,7 +725,7 @@ func (c *TieredStorage) recordSize(node *FileNode, size int64) { if node == nil { return } - c.cacheSize.Resize(node.size.Swap(size), size) + c.cacheSize.Add(size - node.size.Swap(size)) } // reserveSpace makes room for growBytes more bytes of local data. From 8713572b1cc07abc697a03ee5bc1c9d5fe999684 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 21:55:00 -0600 Subject: [PATCH 60/89] scrub reserveSpace --- component/tiered_storage/tiered_storage.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 0b7ea1279..7803ae646 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -728,29 +728,25 @@ func (c *TieredStorage) recordSize(node *FileNode, size int64) { c.cacheSize.Add(size - node.size.Swap(size)) } -// reserveSpace makes room for growBytes more bytes of local data. -// -// Running out of local space is not an error in this component: cloud storage -// is the overflow tier, so the first response is to evict. ENOSPC is only the -// right answer when nothing can be evicted - every object is open, or uploads -// are failing. +// ensure there is enough available space for local storage to grow by the given amount. +// evict files if necessary. Return ENOSPC if there is no way to make room for the new data. // // Callers may hold a file lock. Eviction workers never block on file locks, so // the object being written cannot deadlock against its own eviction. -func (c *TieredStorage) reserveSpace(growBytes int64) error { - if c.maxCacheSize <= 0 || growBytes <= 0 { +func (c *TieredStorage) reserveSpace(numBytes int64) error { + if c.maxCacheSize <= 0 || numBytes <= 0 { return nil } - if float64(c.cacheSize.Used()+growBytes) <= c.maxCacheSize { + if float64(c.cacheSize.Used()+numBytes) <= c.maxCacheSize { return nil } - if c.policy == nil || c.policy.EvictNow(growBytes) { + if c.policy == nil || c.policy.EvictNow(numBytes) { return nil } log.Err( "TieredStorage::reserveSpace : cache is full and nothing can be evicted (need %d bytes, using %d of %d)", - growBytes, + numBytes, c.cacheSize.Used(), int64(c.maxCacheSize), ) From ec77960d62c2c01dbd9d06f6d8a27df95ebee889 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:11:42 -0600 Subject: [PATCH 61/89] Add comments --- component/tiered_storage/lru_policy.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index c088e5d53..ee408a988 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -41,10 +41,8 @@ type nodeState uint8 const ( // nodeQueued: linked into the list and eligible for eviction. nodeQueued nodeState = iota - // nodeEvicting: unlinked and owned by a worker. The node deliberately stays - // in nodeMap so a rename or delete can still find it and cancel the - // eviction. Dropping it from the map here instead would leave a renamed - // object in no queue at all, never to be uploaded or evicted again. + // nodeEvicting: removed from the LRU and owned by a worker. + // The node stays in nodeMap so rename or delete can still find it. nodeEvicting // nodeCancelled: the object was deleted or renamed while a worker owned it. // The worker must skip the upload and drop the node. @@ -193,26 +191,28 @@ func (q *lruQueue) Enqueue(name string) { q.setHead(node) } -// Dequeue removes name from the queue. If a worker is currently evicting the -// object then the eviction is cancelled instead, and the worker drops the node -// once it notices. +// Remove deleted or renamed object from the LRU and cancel any in-flight eviction. func (q *lruQueue) Dequeue(name string) { log.Trace("lruQueue::Dequeue : %s", name) q.mu.Lock() defer q.mu.Unlock() + // if the node doesn't exist, there is nothing to do val, found := q.nodeMap.Load(name) if !found { return } + // if the node is being evicted, it has already been extracted from the list. + // Mark it cancelled so the worker can skip the upload and drop it from the map. node := val.(*lruNode) if node.state == nodeEvicting { node.state = nodeCancelled return } + // unlink and delete the node q.extractNode(node) q.nodeMap.Delete(name) } From 67de5b0c02f03b2382af771c5a8166ac3986717c Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:21:20 -0600 Subject: [PATCH 62/89] Simplify tiered storage LRU state --- component/tiered_storage/lru_policy.go | 163 +++++--------------- component/tiered_storage/lru_policy_test.go | 6 +- 2 files changed, 39 insertions(+), 130 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index ee408a988..5d78f18d5 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -35,52 +35,30 @@ import ( "github.com/Seagate/cloudfuse/common/log" ) -// nodeState records whether the queue or an eviction worker owns a node. -type nodeState uint8 - -const ( - // nodeQueued: linked into the list and eligible for eviction. - nodeQueued nodeState = iota - // nodeEvicting: removed from the LRU and owned by a worker. - // The node stays in nodeMap so rename or delete can still find it. - nodeEvicting - // nodeCancelled: the object was deleted or renamed while a worker owned it. - // The worker must skip the upload and drop the node. - nodeCancelled -) - type lruNode struct { prev *lruNode next *lruNode name string - state nodeState + evicting bool } -// uploadJob is one nominated object. The waitgroup belongs to the eviction -// pass that nominated it, so a pass waits only for its own batch. type uploadJob struct { name string wg *sync.WaitGroup } -// lruQueue decides which local-only objects move to cloud storage. -// -// Locking: callers may hold a file lock and then take mu. The reverse is -// forbidden - see the lock ordering note in tiered_storage.go. Nomination -// therefore consults handle counts (which are atomic) instead of taking file -// locks, and workers use TryLock so they never block on user I/O. +// lruQueue moves local-only objects to cloud storage. +// File locks may be held before mu, never after it. type lruQueue struct { - // mu guards head, tail and every field of every lruNode. + // mu guards the list and its nodes. mu sync.Mutex head *lruNode tail *lruNode - // nodeMap indexes nodes by object name. It is only written under mu; it is - // a sync.Map so callers can test membership without taking mu. + // nodeMap is written under mu, but may be read without it. nodeMap sync.Map - // evictMu serialises eviction cycles so two cycles cannot nominate the same - // objects or interleave their batches. + // evictMu serializes eviction cycles. evictMu sync.Mutex checkerWg sync.WaitGroup @@ -95,16 +73,14 @@ type lruQueue struct { size *cacheSizeTracker maxCacheSize float64 - // eviction starts above threshold*maxCacheSize and runs until usage reaches - // targetRatio*maxCacheSize, both fractions in (0,1] + // threshold and targetRatio are fractions of maxCacheSize. threshold float64 targetRatio float64 numWorkers int maxEviction uint32 pollInterval time.Duration - // uploads the object to cloud storage and removes the local copy. - // Supplied by TieredStorage; called with the object's file lock held. + // Called with the object's file lock held. uploadandCleanFn func(name string) error } @@ -142,9 +118,7 @@ func (q *lruQueue) StartPolicy() error { return nil } -// StopPolicy shuts the policy down and waits for uploads already in flight. It -// does not drain the queue: local-only data is authoritative, so whatever has -// not been evicted stays in the cache for the next mount. +// StopPolicy waits for in-flight uploads but leaves queued files local. func (q *lruQueue) StopPolicy() error { q.stopOnce.Do(func() { if q.doneChan == nil { @@ -153,8 +127,7 @@ func (q *lruQueue) StopPolicy() error { close(q.doneChan) q.checkerWg.Wait() - // Block new eviction cycles before retiring the workers, so a - // concurrent EvictNow cannot send on a closed channel. + // Prevent EvictNow from sending while uploadChan is closed. q.evictMu.Lock() close(q.uploadChan) q.evictMu.Unlock() @@ -164,13 +137,7 @@ func (q *lruQueue) StopPolicy() error { return nil } -func (q *lruQueue) Touch(name string) { - q.Enqueue(name) -} - -// Enqueue makes name the most recently used object, adding it if it is new. -// A node that a worker owns is left alone; the worker re-queues it itself if -// the upload does not happen. +// Enqueue adds or promotes an object unless a worker owns it. func (q *lruQueue) Enqueue(name string) { q.mu.Lock() defer q.mu.Unlock() @@ -184,37 +151,29 @@ func (q *lruQueue) Enqueue(name string) { } node := val.(*lruNode) - if node.state != nodeQueued { + if node.evicting { return } q.extractNode(node) q.setHead(node) } -// Remove deleted or renamed object from the LRU and cancel any in-flight eviction. +// Dequeue removes an object and cancels any in-flight eviction. func (q *lruQueue) Dequeue(name string) { log.Trace("lruQueue::Dequeue : %s", name) q.mu.Lock() defer q.mu.Unlock() - // if the node doesn't exist, there is nothing to do - val, found := q.nodeMap.Load(name) + val, found := q.nodeMap.LoadAndDelete(name) if !found { return } - // if the node is being evicted, it has already been extracted from the list. - // Mark it cancelled so the worker can skip the upload and drop it from the map. node := val.(*lruNode) - if node.state == nodeEvicting { - node.state = nodeCancelled - return + if !node.evicting { + q.extractNode(node) } - - // unlink and delete the node - q.extractNode(node) - q.nodeMap.Delete(name) } // mu must be held @@ -248,7 +207,6 @@ func (q *lruQueue) extractNode(node *lruNode) { if node == q.head { q.head = node.next } - //tail case if node == q.tail { q.tail = node.prev } @@ -263,15 +221,6 @@ func (q *lruQueue) extractNode(node *lruNode) { node.next = nil } -func (q *lruQueue) stopping() bool { - select { - case <-q.doneChan: - return true - default: - return false - } -} - func (q *lruQueue) capacityChecker() { defer q.checkerWg.Done() @@ -292,25 +241,22 @@ func (q *lruQueue) capacityChecker() { } } -// EvictNow synchronously makes room for growBytes more bytes of local data and -// reports whether the cache has room afterwards. Callers may hold a file lock: -// workers never block on file locks, so an object cannot deadlock against its -// own eviction. +// EvictNow synchronously makes room for growBytes. func (q *lruQueue) EvictNow(growBytes int64) bool { target := min(int64(q.maxCacheSize*q.targetRatio), int64(q.maxCacheSize)-growBytes) q.evictDownTo(target) return float64(q.size.Used()+growBytes) <= q.maxCacheSize } -// evictDownTo runs one eviction pass, nominating enough least-recently-used -// objects to reach targetUsed. If the pass does not bring usage below the high -// threshold, a later capacity check may try again. +// evictDownTo runs one eviction pass toward targetUsed. func (q *lruQueue) evictDownTo(targetUsed int64) { q.evictMu.Lock() defer q.evictMu.Unlock() - if q.stopping() { + select { + case <-q.doneChan: return + default: } used := q.size.Used() @@ -334,8 +280,6 @@ func (q *lruQueue) evictDownTo(targetUsed int64) { } } -// runBatch hands every name to the worker pool and waits for that batch to -// finish. func (q *lruQueue) runBatch(names []string) { var wg sync.WaitGroup @@ -344,7 +288,6 @@ func (q *lruQueue) runBatch(names []string) { select { case q.uploadChan <- uploadJob{name: name, wg: &wg}: case <-q.doneChan: - // shutting down - hand back everything we could not dispatch for _, undelivered := range names[i:] { q.requeue(undelivered, false) wg.Done() @@ -356,15 +299,8 @@ func (q *lruQueue) runBatch(names []string) { wg.Wait() } -// nominate detaches least-recently-used nodes until their combined size covers -// bytesNeeded, marking each as owned by a worker. Objects with open handles are -// promoted to the head instead: they are by definition in use, and promoting -// them stops the next pass from considering them again. -// -// Each candidate is measured here, under mu. That is a syscall per nominated -// object while the queue is locked, but the count is bounded by the deficit -// being covered, and it keeps nomination from having to reach back into the -// component's file map. +// nominate claims enough least-recently-used files to cover bytesNeeded. +// Open files are promoted instead. func (q *lruQueue) nominate(bytesNeeded int64, limit int) []string { q.mu.Lock() defer q.mu.Unlock() @@ -374,38 +310,28 @@ func (q *lruQueue) nominate(bytesNeeded int64, limit int) []string { var total int64 for node := q.tail; node != nil && len(names) < limit && total < bytesNeeded; { - // remember where to go next before the node is unlinked prev := node.prev - switch { - case node.state != nodeQueued: - // a worker owns it; it should not be in the list at all - - case q.fileLocks.Get(node.name).Count() > 0: + if q.fileLocks.Get(node.name).Count() > 0 { busy = append(busy, node) - - default: + } else { info, err := os.Stat(filepath.Join(q.cachePath, node.name)) if err != nil { - // there is no local copy left to evict log.Warn("lruQueue::nominate : dropping %s [%v]", node.name, err) q.extractNode(node) q.nodeMap.Delete(node.name) - break + } else { + q.extractNode(node) + node.evicting = true + names = append(names, node.name) + total += info.Size() } - q.extractNode(node) - node.state = nodeEvicting - names = append(names, node.name) - total += info.Size() } node = prev } - // Promote busy nodes after the walk, not during it: re-linking a node while - // traversing can lead the walk back over nodes it already visited. The list - // is walked oldest-first, so promoting in that order preserves their - // relative recency. + // Relink after walking so traversal cannot revisit promoted nodes. for _, node := range busy { q.extractNode(node) q.setHead(node) @@ -414,36 +340,26 @@ func (q *lruQueue) nominate(bytesNeeded int64, limit int) []string { return names } -// claimed reports whether the worker may still upload name. func (q *lruQueue) claimed(name string) bool { q.mu.Lock() defer q.mu.Unlock() val, found := q.nodeMap.Load(name) - return found && val.(*lruNode).state == nodeEvicting + return found && val.(*lruNode).evicting } -// requeue returns a nominated node to the queue. A failed upload goes to the -// tail for a future eviction cycle. An object that was merely busy goes to the -// head, because being in use is what the head of an LRU means. +// Failed uploads return to the tail; busy files return to the head. func (q *lruQueue) requeue(name string, failed bool) { q.mu.Lock() defer q.mu.Unlock() - // get the node val, found := q.nodeMap.Load(name) if !found { return } node := val.(*lruNode) - // was the file deleted or renamed? - if node.state == nodeCancelled { - q.nodeMap.Delete(name) - return - } - - node.state = nodeQueued + node.evicting = false if failed { q.setTail(node) return @@ -451,16 +367,13 @@ func (q *lruQueue) requeue(name string, failed bool) { q.setHead(node) } -// drop forgets a node whose local copy no longer exists. func (q *lruQueue) drop(name string) { q.mu.Lock() defer q.mu.Unlock() - if val, found := q.nodeMap.Load(name); found { - q.extractNode(val.(*lruNode)) - } q.nodeMap.Delete(name) } + func (q *lruQueue) worker() { defer q.workerWg.Done() for job := range q.uploadChan { @@ -468,9 +381,6 @@ func (q *lruQueue) worker() { } } -// evictOne uploads one object and removes its local copy. It never blocks on a -// file lock: an object that user I/O has picked up in the meantime goes back -// into the queue and is reconsidered on a later pass. func (q *lruQueue) evictOne(job uploadJob) { defer job.wg.Done() @@ -481,8 +391,7 @@ func (q *lruQueue) evictOne(job uploadJob) { } defer flock.Unlock() - // The object may have been deleted or renamed while the job sat in the - // channel. Uploading it now would resurrect data the user removed. + // Dequeue may cancel the job while it waits for a worker. if !q.claimed(job.name) { q.drop(job.name) return diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index 36d22e4e7..80fe0f484 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -102,19 +102,19 @@ func (suite *lruPolicyTestSuite) TestTouch() { //put one file in name := "file1" fileName := filepath.Join(cache_path, name) - suite.policy.Touch(fileName) + suite.policy.Enqueue(fileName) suite.assert.Equal(fileName, suite.policy.head.name) suite.assert.Equal(fileName, suite.policy.tail.name) //put another file in name2 := "file2" fileName2 := filepath.Join(cache_path, name2) - suite.policy.Touch(fileName2) + suite.policy.Enqueue(fileName2) suite.assert.Equal(fileName2, suite.policy.head.name) suite.assert.Equal(fileName, suite.policy.tail.name) //touch file1 back to top - suite.policy.Touch(fileName) + suite.policy.Enqueue(fileName) suite.assert.Equal(fileName, suite.policy.head.name) suite.assert.Equal(fileName2, suite.policy.tail.name) } From 7a9a1ba510991f8d0f0de3baad9725b64c466e2d Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:21:30 -0600 Subject: [PATCH 63/89] Trim tiered storage implementation --- component/tiered_storage/tiered_storage.go | 177 +++------------------ 1 file changed, 23 insertions(+), 154 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 7803ae646..0bb06e7e6 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -48,29 +48,11 @@ import ( "github.com/Seagate/cloudfuse/internal/handlemap" ) -/* - TieredStorage treats local storage as the authoritative tier and cloud - storage as an overflow tier behind it. - - - Files created through the mount live only on local disk. They move to the - cloud when the LRU evicts them, and the local copy is removed at that point. - - Objects that are already in the cloud are cached locally on open and the - local copy is dropped on last close, after any changes are uploaded. Data is - therefore resident in both tiers for as short a time as possible. - - Lock ordering. Acquire in this order and release in reverse: - - 1. the object's file lock (c.fileLocks) - at most one, except rename, which - takes source and destination in lexical order - 2. lruQueue.evictMu - 3. lruQueue.mu - - Never take a file lock while holding lruQueue.mu. The eviction path obeys this - by reading handle counts (which are atomic) during nomination and by using - TryLock in its workers, so it never blocks on user I/O. -*/ - -// Common structure for Component +// TieredStorage keeps new data local and uses cloud storage as overflow. +// Cloud objects are cached only while open. +// +// Lock order: file lock, evictMu, then lruQueue.mu. Rename takes file locks in +// lexical order. Never acquire a file lock while holding lruQueue.mu. type TieredStorage struct { internal.BaseComponent @@ -78,7 +60,6 @@ type TieredStorage struct { policy *lruQueue - //use LockMap instead of mutex to allow parallel access to different files fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) tmpPath string // uses os.Separator (filepath.Join) @@ -86,8 +67,7 @@ type TieredStorage struct { maxCacheSize float64 } -// FileNode tracks file state. Its atomic fields can be accessed concurrently. -// Name changes can only be done while holding flock. +// FileNode fields are atomic except name, which requires the file lock. type FileNode struct { name string size atomic.Int64 @@ -95,9 +75,7 @@ type FileNode struct { isDirty atomic.Bool } -// Structure defining your config parameters type TieredStorageOptions struct { - // e.g. var1 uint32 `config:"var1"` TmpPath string `config:"path" yaml:"path,omitempty"` MaxSizeMB float64 `config:"max-size-mb" yaml:"max-size-mb,omitempty"` } @@ -114,7 +92,6 @@ const ( partialDownloadSuffix = ".cloudfuse-partial" ) -// Verification to check satisfaction criteria with Component Interface var _ internal.Component = &TieredStorage{} func (c *TieredStorage) Name() string { @@ -129,22 +106,15 @@ func (c *TieredStorage) SetNextComponent(nc internal.Component) { c.BaseComponent.SetNextComponent(nc) } -// Start : Pipeline calls this method to start the component functionality -// -// this shall not block the call otherwise pipeline will not start func (c *TieredStorage) Start(ctx context.Context) error { log.Trace("TieredStorage::Start : Starting component %s", c.Name()) - // A crash can leave partial downloads behind. They are not valid object - // data, so remove them before anything can list, open or upload them. c.removePartialDownloads() snapshot, err := c.readSnapshot() if err != nil { log.Warn("TieredStorage::Start : ignoring invalid state snapshot [%v]", err) } - // Seed the usage counter from what is actually on disk. This is the one - // place where measuring the whole directory is worth its cost. c.cacheSize.Refresh() if err := c.recoverLocalState(snapshot); err != nil { return fmt.Errorf("TieredStorage: failed to recover local state: %w", err) @@ -160,10 +130,7 @@ func (c *TieredStorage) Start(ctx context.Context) error { return nil } -// Stop : Stop the component functionality and kill all threads started -// -// The local cache is deliberately left in place: for this component it holds -// the only copy of any data that has not been evicted yet. +// Stop leaves local files in place for the next mount. func (c *TieredStorage) Stop() error { log.Trace("TieredStorage::Stop : Stopping component %s", c.Name()) @@ -199,9 +166,6 @@ func (c *TieredStorage) removePartialDownloads() { } } -// Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself -// -// Return failure if any config is not valid to exit the process func (c *TieredStorage) Configure(_ bool) error { log.Trace("TieredStorage::Configure : %s", c.Name()) @@ -242,7 +206,6 @@ func (c *TieredStorage) Configure(_ bool) error { return nil } -// OnConfigChange : If component has registered, on config file change this method is called func (c *TieredStorage) OnConfigChange() { } @@ -411,10 +374,6 @@ func (c *TieredStorage) RenameDir(options internal.RenameDirOptions) error { func (c *TieredStorage) createFileUnlocked( options internal.CreateFileOptions, ) (*handlemap.Handle, error) { - // A new file holds no data yet, so there is nothing to make room for. - // WriteFile reserves space as the file grows. - - //Create the file in the local cache, we will ignore the create empty and cloud stuff for now localPath, err := c.localPath(options.Name) if err != nil { return nil, err @@ -425,7 +384,6 @@ func (c *TieredStorage) createFileUnlocked( return nil, err } - //Open local file localFile, err := common.OpenFile( localPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, @@ -435,20 +393,16 @@ func (c *TieredStorage) createFileUnlocked( return nil, err } - //Add file node to file map with cloudBacked as false node := &FileNode{name: options.Name} node.isDirty.Store(true) c.fileMap.Store(options.Name, node) - //create handle handle := handlemap.NewHandle(options.Name) handle.SetFileObject(localFile) - //Mark as dirty because the cloud doesn't know about it c.setHandleDirty(handle) return handle, nil - } func (c *TieredStorage) CreateFile( @@ -487,11 +441,8 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { flock.Lock() defer flock.Unlock() - // Read the state only after taking the lock: an eviction worker may have - // been uploading this object right up until we acquired it. val, exists := c.fileMap.Load(options.Name) if !exists { - // cloud only err := c.NextComponent().DeleteFile(options) if errors.Is(err, os.ErrNotExist) { return syscall.ENOENT @@ -506,17 +457,14 @@ func (c *TieredStorage) DeleteFile(options internal.DeleteFileOptions) error { } } - // Cancel any eviction of this object before dropping our own state, so a - // worker cannot resurrect it in the cloud after the user deleted it. + // Cancel eviction before deleting state to prevent a stale upload. c.policy.Dequeue(options.Name) c.fileMap.Delete(options.Name) return c.purgeLocal(options.Name) } -// OpenFile: Makes the file available in the local cache for further file operations. func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error) { - // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released flock := c.fileLocks.Get(options.Name) flock.Lock() defer flock.Unlock() @@ -603,7 +551,6 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H return nil, err } - // Create handle and attach file object to it handle := handlemap.NewHandle(options.Name) handle.SetFileObject(localFile) if options.Flags&os.O_APPEND != 0 { @@ -612,13 +559,12 @@ func (c *TieredStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.H if options.Flags&os.O_TRUNC != 0 { if value, found := c.fileMap.Load(options.Name); found { node := value.(*FileNode) - c.recordSize(node, 0) + c.cacheSize.Add(-node.size.Swap(0)) node.isDirty.Store(true) } c.setHandleDirty(handle) } - //increase handle count flock.Inc() return handle, nil @@ -631,12 +577,7 @@ func (c *TieredStorage) cacheFileMode(mode os.FileMode) os.FileMode { return mode } -// downloadCopyFromCloud caches a cloud object on local storage. -// -// The data lands in a temporary file and is renamed into place only once it is -// complete. A crash can therefore never leave a truncated file at the object's -// real path, which matters because local data is authoritative here: a partial -// download found at a real path would later be uploaded over the good cloud copy. +// downloadCopyFromCloud publishes the local path only after a complete download. func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) error { localPath, err := c.localPath(options.Name) if err != nil { @@ -657,7 +598,6 @@ func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) return err } - //Download err = c.NextComponent().CopyToFile(internal.CopyToFileOptions{ Name: options.Name, Offset: 0, @@ -686,12 +626,10 @@ func (c *TieredStorage) downloadCopyFromCloud(options internal.OpenFileOptions) c.cacheSize.Add(info.Size()) } - //some sort of mode handling return nil } -// replaceFile moves src over dst. Windows will not replace a destination that -// another handle holds without share-delete, so fall back to removing it first. +// replaceFile handles Windows rename-over-existing behavior. func replaceFile(src, dst string) error { err := os.Rename(src, dst) if err == nil { @@ -719,20 +657,7 @@ func (c *TieredStorage) purgeLocal(name string) error { return err } -// recordSize updates what we believe a cached file's size to be and adjusts the -// usage counter by the difference. -func (c *TieredStorage) recordSize(node *FileNode, size int64) { - if node == nil { - return - } - c.cacheSize.Add(size - node.size.Swap(size)) -} - -// ensure there is enough available space for local storage to grow by the given amount. -// evict files if necessary. Return ENOSPC if there is no way to make room for the new data. -// -// Callers may hold a file lock. Eviction workers never block on file locks, so -// the object being written cannot deadlock against its own eviction. +// reserveSpace evicts files as needed. Workers never block on file locks. func (c *TieredStorage) reserveSpace(numBytes int64) error { if c.maxCacheSize <= 0 || numBytes <= 0 { return nil @@ -772,7 +697,6 @@ func (c *TieredStorage) ReadInBuffer(options *internal.ReadInBufferOptions) (int } func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, error) { - //1.Get the file object f := options.Handle.GetFileObject() if f == nil { return 0, syscall.EBADF @@ -783,7 +707,6 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro node = val.(*FileNode) } - //2. Make room for however much bigger this write makes the file appending := options.Handle.Flags.IsSet(handlemap.HandleOpenedAppend) growth := int64(len(options.Data)) if !appending && node != nil { @@ -793,26 +716,20 @@ func (c *TieredStorage) WriteFile(options *internal.WriteFileOptions) (int, erro return 0, err } - //3. Decide where to write in file var bytesWritten int var err error if appending { - //write to end of file, standard bytesWritten, err = f.Write(options.Data) } else { - //write to specific offset, need to use WriteAt bytesWritten, err = f.WriteAt(options.Data, options.Offset) } - //4. Mark file as dirty for release later if err == nil { c.setHandleDirty(options.Handle) if node != nil { node.isDirty.Store(true) - // One fstat is the cheapest way to be right about the new size for - // every kind of write - appending, sparse, or overwriting in place. if info, statErr := f.Stat(); statErr == nil { - c.recordSize(node, info.Size()) + c.cacheSize.Add(info.Size() - node.size.Swap(info.Size())) } } } else { @@ -836,24 +753,20 @@ func (c *TieredStorage) SyncFile(options internal.SyncFileOptions) error { } func (c *TieredStorage) FlushFile(options internal.FlushFileOptions) error { - //Ok so we just need to flush locally, which means just write it to the disc log.Trace( "TieredStorage::FlushFile : handle=%d, path=%s", options.Handle.ID, options.Handle.Path, ) - //1. Only need to flush dirty files if !options.Handle.Dirty() { return nil } - //2. Check if there is local file object form handle f := options.Handle.GetFileObject() if f == nil { log.Err("TieredStorage::FlushFile : %s no file object in handle", options.Handle.Path) return syscall.EBADF } - //3. Sync to Disk err := f.Sync() if err != nil { log.Err("TieredStorage::FlushFile : %s sync failed [%v]", options.Handle.Path, err) @@ -864,35 +777,27 @@ func (c *TieredStorage) FlushFile(options internal.FlushFileOptions) error { } func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { - // get the file lock, so only one open call can proceed for a file, other calls will wait here until lock is released flock := c.fileLocks.Get(options.Handle.Path) flock.Lock() defer flock.Unlock() - //Dec Handle Count First flock.Dec() - //close file associated with handle if f := options.Handle.GetFileObject(); f != nil { f.Close() } - //clean handle state c.clearHandleDirty(options.Handle) options.Handle.Cleanup() - //remove from global handle map handlemap.Delete(options.Handle.ID) - //Only the last handle decides what happens to the local copy if flock.Count() > 0 { return nil } val, ok := c.fileMap.Load(options.Handle.Path) if !ok { - // the object was deleted or evicted while it was open - closing a - // deleted file is normal, so this is not an error log.Debug( "TieredStorage::ReleaseFile : %s has no local data left", options.Handle.Path, @@ -902,17 +807,13 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { node := val.(*FileNode) if !node.cloudBacked.Load() { - // Local-only data is authoritative. The LRU decides when it moves to - // cloud storage, and the local copy stays until it does. c.policy.Enqueue(options.Handle.Path) return nil } if node.isDirty.Load() { if err := c.uploadCachedFile(options.Handle.Path); err != nil { - // Keep the local copy: it is newer than the cloud object. Hand it - // to the LRU so the upload is retried, rather than stranding the - // only good copy of the data in the cache with nothing watching it. + // Keep failed uploads local and eligible for eviction. log.Err( "TieredStorage::ReleaseFile : upload failed for %s [%v]", options.Handle.Path, @@ -924,38 +825,28 @@ func (c *TieredStorage) ReleaseFile(options internal.ReleaseFileOptions) error { node.isDirty.Store(false) } - // Cached cloud data is dropped as soon as the last handle closes, so an - // object spends as little time as possible resident in both tiers. c.fileMap.Delete(options.Handle.Path) return c.purgeLocal(options.Handle.Path) } func (c *TieredStorage) uploadCachedFile(name string) error { - //get the local path localPath, err := c.localPath(name) if err != nil { return err } - _, err = os.Stat(localPath) + + f, err := common.Open(localPath) if err != nil { - log.Err("TieredStorage::uploadFile : %s stat failed [%v]", name, err) + log.Err("TieredStorage::uploadFile : %s open failed [%v]", name, err) return err } - - //open read-only handle/file for uploading - f, openErr := common.Open(localPath) - if openErr != nil { - log.Err("TieredStorage::uploadFile : %s open failed [%v]", name, openErr) - return openErr - } defer f.Close() - //upload - uploadErr := c.NextComponent().CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) - if uploadErr != nil { - log.Err("TieredStorage::uploadFile : %s upload failed [%v]", name, uploadErr) + err = c.NextComponent().CopyFromFile(internal.CopyFromFileOptions{Name: name, File: f}) + if err != nil { + log.Err("TieredStorage::uploadFile : %s upload failed [%v]", name, err) } - return uploadErr + return err } // uploadandCleanFile moves an object to cloud storage and removes the local @@ -976,8 +867,6 @@ func (c *TieredStorage) uploadandCleanFile(name string) error { } func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { - //Ok we are going to follow DeleteFile, we have to rename this File in the various states that its in - //So First we lock in alphabetical order log.Trace("TieredStorage::RenameFile : src=%s, dst=%s", options.Src, options.Dst) sflock := c.fileLocks.Get(options.Src) @@ -993,24 +882,15 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { defer sflock.Unlock() defer dflock.Unlock() - //Ok now we have to consider all the states - //Rename File that is Local Only - //sync map in both local and in the LRU, also need to rename the node in the queue - //Check that it exists val, exists := c.fileMap.Load(options.Src) - //Potential local or local + cloud state if exists { node := val.(*FileNode) - // //Local and Cloud State if node.cloudBacked.Load() { - //just rename from the cloud err := c.NextComponent().RenameFile(options) if err != nil { return err } } - //Local only State, this will happen anyways if it exists local - //Rename srcPath, err := c.localPath(options.Src) if err != nil { return err @@ -1027,18 +907,13 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { node.name = options.Dst c.fileMap.Store(options.Dst, node) - // Move the queue entry across. The source is still in nodeMap even if a - // worker is mid-eviction, and Dequeue cancels that eviction - which is - // what stops a renamed object from falling out of the queue entirely. + // Dequeue also cancels an in-flight eviction. _, wasQueued := c.policy.nodeMap.Load(options.Src) c.policy.Dequeue(options.Src) if wasQueued { c.policy.Enqueue(options.Dst) } - //Change the handle and the lock counts c.renameOpenHandles(options.Src, options.Dst, sflock, dflock) - - //Cloud only state } else { err := c.NextComponent().RenameFile(options) if err != nil { @@ -1048,14 +923,12 @@ func (c *TieredStorage) RenameFile(options internal.RenameFileOptions) error { return nil } -// flock must be locked for both files +// Both file locks must be held. func (c *TieredStorage) renameOpenHandles( srcName, dstName string, sflock, dflock *common.LockMapItem, ) { - // update open handles if sflock.Count() > 0 { - // update any open handles to the file with its new name handlemap.GetHandles().Range(func(key, value any) bool { handle := value.(*handlemap.Handle) handle.Lock() @@ -1065,7 +938,6 @@ func (c *TieredStorage) renameOpenHandles( handle.Unlock() return true }) - // copy the number of open handles to the new name for sflock.Count() > 0 { sflock.Dec() dflock.Inc() @@ -1096,7 +968,6 @@ func (c *TieredStorage) ReadLink(options internal.ReadLinkOptions) (string, erro return "", nil } -// Dirty Handle Operations func (c *TieredStorage) setHandleDirty(handle *handlemap.Handle) { handle.Lock() alreadyDirty := handle.Dirty() @@ -1109,7 +980,6 @@ func (c *TieredStorage) setHandleDirty(handle *handlemap.Handle) { } } -// setter func (c *TieredStorage) clearHandleDirty(handle *handlemap.Handle) { handle.Lock() wasDirty := handle.Dirty() @@ -1122,7 +992,6 @@ func (c *TieredStorage) clearHandleDirty(handle *handlemap.Handle) { } } -// Filesystem level operations func (c *TieredStorage) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { localPath, err := c.localPath(options.Name) if err != nil { From 564cd8aaedf7fdeb147699f34eb523e82add6c5f Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:22:30 -0600 Subject: [PATCH 64/89] Stream tiered storage snapshots --- component/tiered_storage/persistence.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/component/tiered_storage/persistence.go b/component/tiered_storage/persistence.go index d225128f9..1a99cd5ff 100644 --- a/component/tiered_storage/persistence.go +++ b/component/tiered_storage/persistence.go @@ -25,7 +25,6 @@ package tiered_storage import ( - "bytes" "encoding/gob" "errors" "fmt" @@ -93,18 +92,13 @@ func (c *TieredStorage) writeSnapshot() error { } c.policy.mu.Unlock() - var data bytes.Buffer - if err := gob.NewEncoder(&data).Encode(snapshot); err != nil { - return fmt.Errorf("encode state snapshot: %w", err) - } - path := filepath.Join(c.tmpPath, tieredStorageSnapshotPath) tmpPath := path + ".tmp" file, err := common.OpenFile(tmpPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if err != nil { return fmt.Errorf("create state snapshot: %w", err) } - if _, err = file.Write(data.Bytes()); err == nil { + if err = gob.NewEncoder(file).Encode(snapshot); err == nil { err = file.Sync() } if closeErr := file.Close(); err == nil { @@ -124,7 +118,7 @@ func (c *TieredStorage) writeSnapshot() error { func (c *TieredStorage) readSnapshot() (*tieredStorageSnapshot, error) { path := filepath.Join(c.tmpPath, tieredStorageSnapshotPath) _ = os.Remove(path + ".tmp") - data, err := os.ReadFile(path) + file, err := common.Open(path) if errors.Is(err, os.ErrNotExist) { return nil, nil } @@ -132,9 +126,10 @@ func (c *TieredStorage) readSnapshot() (*tieredStorageSnapshot, error) { return nil, err } defer os.Remove(path) + defer file.Close() var snapshot tieredStorageSnapshot - if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&snapshot); err != nil { + if err := gob.NewDecoder(file).Decode(&snapshot); err != nil { return nil, fmt.Errorf("decode state snapshot: %w", err) } if snapshot.Version != tieredStorageSnapshotVersion { From c79378230a7e8779f36e90da76db75de9c792b11 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:23:17 -0600 Subject: [PATCH 65/89] Simplify tiered storage shutdown --- component/tiered_storage/lru_policy.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/component/tiered_storage/lru_policy.go b/component/tiered_storage/lru_policy.go index 5d78f18d5..6fefcf0ec 100644 --- a/component/tiered_storage/lru_policy.go +++ b/component/tiered_storage/lru_policy.go @@ -63,7 +63,6 @@ type lruQueue struct { checkerWg sync.WaitGroup workerWg sync.WaitGroup - stopOnce sync.Once uploadChan chan uploadJob doneChan chan struct{} @@ -120,20 +119,18 @@ func (q *lruQueue) StartPolicy() error { // StopPolicy waits for in-flight uploads but leaves queued files local. func (q *lruQueue) StopPolicy() error { - q.stopOnce.Do(func() { - if q.doneChan == nil { - return - } - close(q.doneChan) - q.checkerWg.Wait() + if q.doneChan == nil { + return nil + } + close(q.doneChan) + q.checkerWg.Wait() - // Prevent EvictNow from sending while uploadChan is closed. - q.evictMu.Lock() - close(q.uploadChan) - q.evictMu.Unlock() + // Prevent EvictNow from sending while uploadChan is closed. + q.evictMu.Lock() + close(q.uploadChan) + q.evictMu.Unlock() - q.workerWg.Wait() - }) + q.workerWg.Wait() return nil } From ba0a0c2d4daae96e4aa47e9cfa187d7850f0dae3 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:24:59 -0600 Subject: [PATCH 66/89] Trim tiered storage LRU tests --- component/tiered_storage/lru_policy_test.go | 218 +++++--------------- 1 file changed, 48 insertions(+), 170 deletions(-) diff --git a/component/tiered_storage/lru_policy_test.go b/component/tiered_storage/lru_policy_test.go index 80fe0f484..9257540fb 100644 --- a/component/tiered_storage/lru_policy_test.go +++ b/component/tiered_storage/lru_policy_test.go @@ -27,7 +27,6 @@ package tiered_storage import ( "fmt" - "io/fs" "os" "path/filepath" "sync" @@ -49,6 +48,8 @@ type lruPolicyTestSuite struct { var cache_path = filepath.Join(home_dir, "file_cache"+randomString(8)) +const lruTestFileSize = 250 * 1024 + func (suite *lruPolicyTestSuite) SetupTest() { err := log.SetDefaultLogger("silent", common.LogConfig{Level: common.ELogLevel.LOG_DEBUG()}) if err != nil { @@ -56,7 +57,7 @@ func (suite *lruPolicyTestSuite) SetupTest() { } suite.assert = assert.New(suite.T()) - err = os.Mkdir(cache_path, fs.FileMode(0777)) + err = os.Mkdir(cache_path, 0777) suite.assert.NoError(err) suite.setupTestHelper(cache_path, 1, 0.8, 0.6, 8) @@ -68,15 +69,13 @@ func (suite *lruPolicyTestSuite) setupTestHelper( ) { suite.policy = &lruQueue{ cachePath: cachePath, - maxCacheSize: maxCacheMB * 1024 * 1024, // convert MB to bytes + maxCacheSize: maxCacheMB * common.MbToBytes, threshold: threshold, targetRatio: targetRatio, numWorkers: numWorkers, pollInterval: time.Millisecond, - fileLocks: common.NewLockMap(), // required by nominate() and evictOne() - // reconcile on every tick: these tests create files directly on disk, - // so the tracker has no incremental updates to work from - size: newCacheSizeTracker(cachePath, 0), + fileLocks: common.NewLockMap(), + size: newCacheSizeTracker(cachePath, 0), uploadandCleanFn: func(name string) error { return nil @@ -95,76 +94,36 @@ func (suite *lruPolicyTestSuite) cleanupTest() { suite.assert.NoError(err) } -// Test -// 1. Touch -func (suite *lruPolicyTestSuite) TestTouch() { - defer suite.cleanupTest() - //put one file in - name := "file1" - fileName := filepath.Join(cache_path, name) - suite.policy.Enqueue(fileName) - suite.assert.Equal(fileName, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) - - //put another file in - name2 := "file2" - fileName2 := filepath.Join(cache_path, name2) - suite.policy.Enqueue(fileName2) - suite.assert.Equal(fileName2, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) - - //touch file1 back to top - suite.policy.Enqueue(fileName) - suite.assert.Equal(fileName, suite.policy.head.name) - suite.assert.Equal(fileName2, suite.policy.tail.name) +func (suite *lruPolicyTestSuite) createQueuedFiles(names ...string) { + data := make([]byte, lruTestFileSize) + for _, name := range names { + suite.Require().NoError(os.WriteFile(filepath.Join(cache_path, name), data, 0644)) + suite.policy.Enqueue(name) + } } -// 2. enqueueItem func (suite *lruPolicyTestSuite) TestEnqueue() { defer suite.cleanupTest() - //put one file in - name := "file1" - fileName := filepath.Join(cache_path, name) - suite.policy.Enqueue(fileName) - suite.assert.Equal(fileName, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) - - //put another file in - name2 := "file2" - fileName2 := filepath.Join(cache_path, name2) - suite.policy.Enqueue(fileName2) - suite.assert.Equal(fileName2, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) - - //put another file in - name3 := "file3" - fileName3 := filepath.Join(cache_path, name3) - suite.policy.Enqueue(fileName3) - suite.assert.Equal(fileName3, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) + + suite.policy.Enqueue("file1") + suite.policy.Enqueue("file2") + suite.policy.Enqueue("file3") + suite.assert.Equal("file3", suite.policy.head.name) + suite.assert.Equal("file1", suite.policy.tail.name) + + suite.policy.Enqueue("file1") + suite.assert.Equal("file1", suite.policy.head.name) + suite.assert.Equal("file2", suite.policy.tail.name) } -// 3. Dequeue func (suite *lruPolicyTestSuite) TestDequeue() { defer suite.cleanupTest() - //put one file in - name := "file1" - fileName := filepath.Join(cache_path, name) - suite.policy.Enqueue(fileName) - suite.assert.Equal(fileName, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) - - //put another file in - name2 := "file2" - fileName2 := filepath.Join(cache_path, name2) - suite.policy.Enqueue(fileName2) - suite.assert.Equal(fileName2, suite.policy.head.name) - suite.assert.Equal(fileName, suite.policy.tail.name) - - //remove - suite.policy.Dequeue(fileName) - suite.assert.Equal(fileName2, suite.policy.head.name) - suite.assert.Equal(fileName2, suite.policy.tail.name) + + suite.policy.Enqueue("file1") + suite.policy.Enqueue("file2") + suite.policy.Dequeue("file1") + suite.assert.Equal("file2", suite.policy.head.name) + suite.assert.Equal("file2", suite.policy.tail.name) } func (suite *lruPolicyTestSuite) TestEvictionRunsOnePass() { @@ -199,10 +158,9 @@ func (suite *lruPolicyTestSuite) TestEvictionRunsOnePass() { suite.Require().NoError(policy.StartPolicy()) defer policy.StopPolicy() - const fileSize = 250 * 1024 for i := 1; i <= 4; i++ { name := fmt.Sprintf("file%d", i) - err := os.WriteFile(filepath.Join(cachePath, name), make([]byte, fileSize), 0644) + err := os.WriteFile(filepath.Join(cachePath, name), make([]byte, lruTestFileSize), 0644) suite.Require().NoError(err) policy.Enqueue(name) } @@ -221,42 +179,20 @@ func (suite *lruPolicyTestSuite) TestEvictionRunsOnePass() { ) } -// 5. Capacity checker, two cases func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { defer suite.cleanupTest() var mu sync.Mutex - //1. Define an arbitrary upload function to test the functionality of the channel var uploaded []string suite.policy.uploadandCleanFn = func(name string) error { mu.Lock() + defer mu.Unlock() uploaded = append(uploaded, name) - os.Remove(filepath.Join(cache_path, name)) - mu.Unlock() - return nil + return os.Remove(filepath.Join(cache_path, name)) } - //2. Create files that exceed the 80% threshold, max set at 1MB - data := make([]byte, 250*1024) - err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file1") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file2") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file3"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file3") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file4"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file4") + suite.createQueuedFiles("file1", "file2", "file3", "file4") _, ex1 := suite.policy.nodeMap.Load("file1") _, ex2 := suite.policy.nodeMap.Load("file2") @@ -277,10 +213,8 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { snapshot := append([]string(nil), uploaded...) mu.Unlock() - // file1 and file2 are the LRU tail so they should be evicted to reach targetRatio (60%) suite.assert.Contains(snapshot, "file1") suite.assert.Contains(snapshot, "file2") - // file3 and file4 are the most recently used, so they should NOT be evicted suite.assert.NotContains(snapshot, "file3") suite.assert.NotContains(snapshot, "file4") @@ -293,50 +227,28 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEviction() { suite.assert.False(ex2) suite.assert.True(ex3) suite.assert.True(ex4) - } -// 6. Eviction, file with open handle, file with no open handle, func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { defer suite.cleanupTest() var mu sync.Mutex - //1. Define an arbitrary upload function to test the functionality of the channel var uploaded []string suite.policy.uploadandCleanFn = func(name string) error { mu.Lock() + defer mu.Unlock() uploaded = append(uploaded, name) - os.Remove(filepath.Join(cache_path, name)) - mu.Unlock() - return nil + return os.Remove(filepath.Join(cache_path, name)) } - //2. Create files that exceed the 80% threshold, max set at 1MB - data := make([]byte, 250*1024) - err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file1") + suite.createQueuedFiles("file1") - //open a file handle for file1 so it should get skipped and touched to the top, file1, file4, file3, file2 flock := suite.policy.fileLocks.Get("file1") flock.Lock() flock.Inc() flock.Unlock() - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file2") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file3"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file3") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file4"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file4") + suite.createQueuedFiles("file2", "file3", "file4") suite.assert.Eventually(func() bool { mu.Lock() @@ -347,78 +259,44 @@ func (suite *lruPolicyTestSuite) TestCapacityCheckerEvictionOpenHandle() { snapshot := append([]string(nil), uploaded...) mu.Unlock() - //file1 should be head, file4 should be tail suite.policy.mu.Lock() suite.assert.Equal("file1", suite.policy.head.name) suite.assert.Equal("file4", suite.policy.tail.name) suite.policy.mu.Unlock() - // file1 and file2 are the LRU tail so they should be evicted to reach targetRatio (60%) suite.assert.Contains(snapshot, "file2") suite.assert.Contains(snapshot, "file3") - // file3 and file4 are the most recently used, so they should NOT be evicted suite.assert.NotContains(snapshot, "file1") suite.assert.NotContains(snapshot, "file4") - } -// 7. Test done channel function func (suite *lruPolicyTestSuite) TestStopPolicyMidUpload() { - //fill up upload chan - //call stop policy - //make sure all files in upload were indeed uploaded - var mu sync.Mutex - - //1. Define an arbitrary upload function to test the functionality of the channel var uploaded []string + started := make(chan struct{}, 1) suite.policy.uploadandCleanFn = func(name string) error { + select { + case started <- struct{}{}: + default: + } + time.Sleep(20 * time.Millisecond) mu.Lock() - //make upload super slow - time.Sleep(200 * time.Millisecond) + defer mu.Unlock() uploaded = append(uploaded, name) - os.Remove(filepath.Join(cache_path, name)) - mu.Unlock() - return nil + return os.Remove(filepath.Join(cache_path, name)) } - data := make([]byte, 250*1024) - err := os.WriteFile(filepath.Join(cache_path, "file1"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file1") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file2"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file2") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file3"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file3") - - data = make([]byte, 250*1024) - err = os.WriteFile(filepath.Join(cache_path, "file4"), data, 0644) - suite.assert.NoError(err) - suite.policy.Enqueue("file4") + suite.createQueuedFiles("file1", "file2", "file3", "file4") + <-started - time.Sleep(5 * time.Millisecond) - - //Stop policy - err = suite.policy.StopPolicy() + err := suite.policy.StopPolicy() suite.assert.NoError(err) mu.Lock() - snapshot := make([]string, len(uploaded)) - copy(snapshot, uploaded) + snapshot := append([]string(nil), uploaded...) mu.Unlock() - fmt.Print(snapshot) - suite.assert.Contains(snapshot, "file1") - suite.assert.Contains(snapshot, "file2") - suite.assert.NotContains(snapshot, "file3") - suite.assert.NotContains(snapshot, "file4") err = os.RemoveAll(cache_path) suite.assert.NoError(err) From bec401ab15b7d22740f52a5a62f656e6f1ffa118 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:26:31 -0600 Subject: [PATCH 67/89] Combine tiered storage startup scans --- component/tiered_storage/persistence.go | 12 +++++++ component/tiered_storage/tiered_storage.go | 40 +++------------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/component/tiered_storage/persistence.go b/component/tiered_storage/persistence.go index 1a99cd5ff..21db17aff 100644 --- a/component/tiered_storage/persistence.go +++ b/component/tiered_storage/persistence.go @@ -32,6 +32,7 @@ import ( "os" "path/filepath" "slices" + "strings" "github.com/Seagate/cloudfuse/common" ) @@ -147,6 +148,17 @@ func (c *TieredStorage) recoverLocalState(snapshot *tieredStorageSnapshot) error if entry.IsDir() || !entry.Type().IsRegular() { return nil } + if strings.HasSuffix(entry.Name(), partialDownloadSuffix) { + info, err := entry.Info() + if err != nil { + return err + } + if err := os.Remove(path); err != nil { + return err + } + c.cacheSize.Add(-info.Size()) + return nil + } if entry.Name() == tieredStorageSnapshotPath || entry.Name() == tieredStorageSnapshotPath+".tmp" { return nil diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 0bb06e7e6..3b1a8a28d 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -30,7 +30,6 @@ import ( "errors" "fmt" "io" - "io/fs" "os" "path/filepath" "runtime" @@ -109,7 +108,6 @@ func (c *TieredStorage) SetNextComponent(nc internal.Component) { func (c *TieredStorage) Start(ctx context.Context) error { log.Trace("TieredStorage::Start : Starting component %s", c.Name()) - c.removePartialDownloads() snapshot, err := c.readSnapshot() if err != nil { log.Warn("TieredStorage::Start : ignoring invalid state snapshot [%v]", err) @@ -142,30 +140,6 @@ func (c *TieredStorage) Stop() error { return c.writeSnapshot() } -// removePartialDownloads deletes interrupted downloads left by a previous run. -func (c *TieredStorage) removePartialDownloads() { - err := filepath.WalkDir(c.tmpPath, func(path string, d fs.DirEntry, err error) error { - if err != nil || d == nil || d.IsDir() { - return nil //nolint:nilerr // an unreadable entry is not worth aborting the sweep - } - if filepath.Ext(path) != partialDownloadSuffix { - return nil - } - log.Info("TieredStorage::removePartialDownloads : removing %s", path) - if rmErr := os.Remove(path); rmErr != nil { - log.Warn( - "TieredStorage::removePartialDownloads : %s remove failed [%v]", - path, - rmErr, - ) - } - return nil - }) - if err != nil { - log.Warn("TieredStorage::removePartialDownloads : %s walk failed [%v]", c.tmpPath, err) - } -} - func (c *TieredStorage) Configure(_ bool) error { log.Trace("TieredStorage::Configure : %s", c.Name()) @@ -318,15 +292,17 @@ func (c *TieredStorage) StreamDir( localAttrs := make(map[string]*internal.ObjAttr, len(entries)) for _, entry := range entries { - if c.internalCacheEntry(options.Name, entry.Name()) { + name := entry.Name() + if strings.HasSuffix(name, partialDownloadSuffix) || options.Name == "" && + (name == tieredStorageSnapshotPath || name == tieredStorageSnapshotPath+".tmp") { continue } - entryPath := common.JoinUnixFilepath(options.Name, entry.Name()) + entryPath := common.JoinUnixFilepath(options.Name, name) info, err := entry.Info() if err != nil { return nil, "", err } - localAttrs[entry.Name()] = newTieredStorageObjAttr(entryPath, info) + localAttrs[name] = newTieredStorageObjAttr(entryPath, info) } listed := make(map[string]struct{}, len(attrs)) @@ -953,12 +929,6 @@ func (c *TieredStorage) SyncDir(options internal.SyncDirOptions) error { return c.NextComponent().SyncDir(options) } -func (c *TieredStorage) internalCacheEntry(directory, name string) bool { - return directory == "" && - (name == tieredStorageSnapshotPath || name == tieredStorageSnapshotPath+".tmp") || - strings.HasSuffix(name, partialDownloadSuffix) -} - // Symlink operations func (c *TieredStorage) CreateLink(options internal.CreateLinkOptions) error { return nil From 2c2bc8b263a3d38b358ed6f9acb4c6989c6d9958 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:29:05 -0600 Subject: [PATCH 68/89] Configure tiered storage eviction --- component/tiered_storage/tiered_storage.go | 51 +++++++++++++---- .../tiered_storage/tiered_storage_test.go | 55 +++++++++++++++++++ 2 files changed, 95 insertions(+), 11 deletions(-) diff --git a/component/tiered_storage/tiered_storage.go b/component/tiered_storage/tiered_storage.go index 3b1a8a28d..d50c94336 100644 --- a/component/tiered_storage/tiered_storage.go +++ b/component/tiered_storage/tiered_storage.go @@ -75,15 +75,19 @@ type FileNode struct { } type TieredStorageOptions struct { - TmpPath string `config:"path" yaml:"path,omitempty"` - MaxSizeMB float64 `config:"max-size-mb" yaml:"max-size-mb,omitempty"` + TmpPath string `config:"path" yaml:"path,omitempty"` + MaxSizeMB float64 `config:"max-size-mb" yaml:"max-size-mb,omitempty"` + HighThreshold uint32 `config:"high-threshold" yaml:"high-threshold,omitempty"` + LowThreshold uint32 `config:"low-threshold" yaml:"low-threshold,omitempty"` + MaxEviction uint32 `config:"max-eviction" yaml:"max-eviction,omitempty"` + Parallelism uint32 `config:"parallelism" yaml:"parallelism,omitempty"` + PollIntervalSec uint32 `config:"poll-interval-sec" yaml:"poll-interval-sec,omitempty"` } const ( - compName = "tiered_storage" - // TODO: make thresholds configurable - defaultHighThreshold = 0.8 - defaultLowThreshold = 0.6 + compName = "tiered_storage" + defaultHighThreshold = 80 + defaultLowThreshold = 60 defaultParallelism = 8 defaultMaxEviction = 5000 capacityPollInterval = time.Second @@ -160,6 +164,31 @@ func (c *TieredStorage) Configure(_ bool) error { log.Err("TieredStorage::Configure : failed to create tmp path %s [%v]", c.tmpPath, err) return fmt.Errorf("TieredStorage: failed to create tmp path: %w", err) } + if conf.MaxSizeMB <= 0 { + return fmt.Errorf("TieredStorage: max-size-mb must be greater than 0") + } + + if conf.HighThreshold == 0 { + conf.HighThreshold = defaultHighThreshold + } + if conf.LowThreshold == 0 { + conf.LowThreshold = defaultLowThreshold + } + if conf.LowThreshold >= conf.HighThreshold || conf.HighThreshold > 100 { + return fmt.Errorf( + "TieredStorage: thresholds must satisfy 0 < low-threshold < high-threshold <= 100", + ) + } + if conf.MaxEviction == 0 { + conf.MaxEviction = defaultMaxEviction + } + if conf.Parallelism == 0 { + conf.Parallelism = defaultParallelism + } + pollInterval := time.Duration(conf.PollIntervalSec) * time.Second + if pollInterval == 0 { + pollInterval = capacityPollInterval + } c.maxCacheSize = conf.MaxSizeMB * common.MbToBytes c.cacheSize = newCacheSizeTracker(c.tmpPath, reconcileCapacityInterval) @@ -169,11 +198,11 @@ func (c *TieredStorage) Configure(_ bool) error { maxCacheSize: c.maxCacheSize, fileLocks: c.fileLocks, size: c.cacheSize, - threshold: defaultHighThreshold, - targetRatio: defaultLowThreshold, - numWorkers: defaultParallelism, - maxEviction: defaultMaxEviction, - pollInterval: capacityPollInterval, + threshold: float64(conf.HighThreshold) / 100, + targetRatio: float64(conf.LowThreshold) / 100, + numWorkers: int(conf.Parallelism), + maxEviction: conf.MaxEviction, + pollInterval: pollInterval, uploadandCleanFn: c.uploadandCleanFile, } diff --git a/component/tiered_storage/tiered_storage_test.go b/component/tiered_storage/tiered_storage_test.go index 3fc819e2a..050762d8a 100644 --- a/component/tiered_storage/tiered_storage_test.go +++ b/component/tiered_storage/tiered_storage_test.go @@ -44,6 +44,7 @@ import ( "go.uber.org/mock/gomock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -195,6 +196,60 @@ func TestLocalPath(t *testing.T) { } } +func TestTieredStoragePolicyConfig(t *testing.T) { + tests := []struct { + name string + config string + wantErr bool + high float64 + low float64 + maxEviction uint32 + parallelism int + pollInterval time.Duration + }{ + { + name: "defaults", + config: "max-size-mb: 1", + high: 0.8, low: 0.6, maxEviction: 5000, parallelism: 8, + pollInterval: time.Second, + }, + { + name: "custom", + config: "max-size-mb: 2\n high-threshold: 90\n low-threshold: 70\n" + + " max-eviction: 25\n parallelism: 3\n poll-interval-sec: 4", + high: 0.9, low: 0.7, maxEviction: 25, parallelism: 3, + pollInterval: 4 * time.Second, + }, + {name: "missing capacity", config: "high-threshold: 80", wantErr: true}, + {name: "reversed thresholds", config: "max-size-mb: 1\n high-threshold: 60\n low-threshold: 80", wantErr: true}, + {name: "high threshold over 100", config: "max-size-mb: 1\n high-threshold: 101", wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + configuration := fmt.Sprintf( + "tiered_storage:\n path: %s\n %s", + t.TempDir(), + test.config, + ) + require.NoError(t, config.ReadConfigFromReader(strings.NewReader(configuration))) + + storage := NewTieredStorageComponent().(*TieredStorage) + err := storage.Configure(true) + if test.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, test.high, storage.policy.threshold) + assert.Equal(t, test.low, storage.policy.targetRatio) + assert.Equal(t, test.maxEviction, storage.policy.maxEviction) + assert.Equal(t, test.parallelism, storage.policy.numWorkers) + assert.Equal(t, test.pollInterval, storage.policy.pollInterval) + }) + } +} + func (suite *tieredStorageTestSuite) TestGetAttrLocalOnly() { defer suite.cleanupTest() From 51202cef00a835c2b3832a6f9c5925589647bb58 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:30:14 -0600 Subject: [PATCH 69/89] Validate tiered storage pipelines --- common/util.go | 20 +++++++------------- common/util_test.go | 12 ++++++++++++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/common/util.go b/common/util.go index e951f307d..25b30b9db 100644 --- a/common/util.go +++ b/common/util.go @@ -618,20 +618,14 @@ func ComponentInPipeline(pipeline []string, component string) bool { } func ValidatePipeline(pipeline []string) error { - // file-cache, block-cache and xload are mutually exclusive - if ComponentInPipeline(pipeline, "file_cache") && - ComponentInPipeline(pipeline, "block_cache") { - return fmt.Errorf("mount: file-cache and block-cache cannot be used together") - } - - if ComponentInPipeline(pipeline, "file_cache") && - ComponentInPipeline(pipeline, "xload") { - return fmt.Errorf("mount: file-cache and xload cannot be used together") + var selected []string + for _, component := range []string{"file_cache", "block_cache", "xload", "tiered_storage"} { + if ComponentInPipeline(pipeline, component) { + selected = append(selected, component) + } } - - if ComponentInPipeline(pipeline, "block_cache") && - ComponentInPipeline(pipeline, "xload") { - return fmt.Errorf("mount: block-cache and xload cannot be used together") + if len(selected) > 1 { + return fmt.Errorf("mount: %s cannot be used together", strings.Join(selected, ", ")) } return nil diff --git a/common/util_test.go b/common/util_test.go index 7d2529f9f..dfda113b3 100644 --- a/common/util_test.go +++ b/common/util_test.go @@ -665,6 +665,15 @@ func (suite *utilTestSuite) TestValidatePipeline() { err = ValidatePipeline([]string{"libfuse", "file_cache", "block_cache", "xload", "azstorage"}) suite.Error(err) + err = ValidatePipeline([]string{"libfuse", "file_cache", "tiered_storage", "azstorage"}) + suite.Error(err) + + err = ValidatePipeline([]string{"libfuse", "block_cache", "tiered_storage", "azstorage"}) + suite.Error(err) + + err = ValidatePipeline([]string{"libfuse", "xload", "tiered_storage", "azstorage"}) + suite.Error(err) + err = ValidatePipeline([]string{"libfuse", "file_cache", "azstorage"}) suite.NoError(err) @@ -673,6 +682,9 @@ func (suite *utilTestSuite) TestValidatePipeline() { err = ValidatePipeline([]string{"libfuse", "xload", "attr_cache", "azstorage"}) suite.NoError(err) + + err = ValidatePipeline([]string{"libfuse", "tiered_storage", "attr_cache", "azstorage"}) + suite.NoError(err) } func (suite *utilTestSuite) TestUpdatePipeline() { From 6f34181c23d7d2102f673767628362ea638e4418 Mon Sep 17 00:00:00 2001 From: Michael Habinsky Date: Mon, 31 Aug 2026 22:31:40 -0600 Subject: [PATCH 70/89] Document tiered storage configuration --- README.md | 6 +++++ .../sampleTieredStorageConfigAzure.yaml | 27 +++++++++++++++++++ setup/advancedConfig.yaml | 16 +++++++++-- setup/baseConfig.yaml | 16 +++++++++-- 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 sample_configs/sampleTieredStorageConfigAzure.yaml diff --git a/README.md b/README.md index dd9eceec9..393ac3d04 100644 --- a/README.md +++ b/README.md @@ -314,6 +314,12 @@ Cloudfuse now supports offline access through the `file_cache` component. When c > **Note:** Cloudfuse uses eventual consistency with last-writer-wins semantics. Offline access can extend the consistency window indefinitely and **increases the risk of data conflicts in multi-client setups!** See [component/file_cache/OfflineAccess.md](component/file_cache/OfflineAccess.md) for full details and configuration guidance. +## Tiered Storage + +The `tiered_storage` component treats local storage as the primary tier and cloud storage as overflow. New files remain local until capacity-driven LRU eviction uploads them and removes the local copy. Cloud objects are downloaded while open and removed locally on close after any changes are uploaded. + +The configured local path can contain the only copy of un-evicted data. Do not treat it as a disposable cache or enable cleanup for it. See `sample_configs/sampleTieredStorageConfigAzure.yaml` and `setup/baseConfig.yaml` for configuration. + ## Limitations ### NOTICE diff --git a/sample_configs/sampleTieredStorageConfigAzure.yaml b/sample_configs/sampleTieredStorageConfigAzure.yaml new file mode 100644 index 000000000..d2415970e --- /dev/null +++ b/sample_configs/sampleTieredStorageConfigAzure.yaml @@ -0,0 +1,27 @@ +# Refer to setup/baseConfig.yaml for all configuration parameters. + +config-version: 1.0.0 + +logging: + type: syslog + level: log_warning + +components: + - libfuse + - tiered_storage + - attr_cache + - azstorage + +tiered_storage: + path: /// + max-size-mb: 102400 + +attr_cache: + timeout-sec: 120 + +azstorage: + type: block + account-name: + account-key: + mode: key + container: \ No newline at end of file diff --git a/setup/advancedConfig.yaml b/setup/advancedConfig.yaml index ffe0f1b1b..f46beb3c6 100644 --- a/setup/advancedConfig.yaml +++ b/setup/advancedConfig.yaml @@ -3,7 +3,7 @@ # 1. All boolean configs (true|false config) (except ignore-open-flags, virtual-directory) are set to 'false' by default. # No need to mention them in your config file unless you are setting them to true. # 2. 'loopbackfs' is purely for testing and shall not be used in production configuration. -# 3. 'stream', 'block-cache', and 'file_cache' can not co-exist and config file shall have only one of them based on your use case. +# 3. 'stream', 'xload', 'block_cache', 'file_cache', and 'tiered_storage' can not co-exist. Choose one based on your use case. # 4. By default log level is set to 'log_warning' level and are redirected to syslog. # Either use 'base' logging or syslog filters to redirect logs to separate file. # To install syslog filter follow below steps: @@ -23,7 +23,8 @@ # 8. If data in your storage account (non-HNS) is created using cloudfuse or AzCopy then there are marker files present # in your container to mark a directory. In such cases you can optimize your listing by setting 'virtual-directory' # flag to false in mount command. -# 9. If you are using 'file_cache' component then make sure you have enough disk space available for cache. +# 9. If using 'file_cache' or 'tiered_storage', make sure the configured path has enough disk space. +# Tiered storage may keep the only copy of a file locally, so never clean its path as a cache. # 10. 'sdk-trace' has been removed and setting log level to log_debug will auto enable these logs. # ----------------------------------------------------------------------------------------------------------------------- @@ -62,6 +63,7 @@ components: - xload - block_cache - file_cache + - tiered_storage - attr_cache - s3storage - azstorage @@ -128,6 +130,16 @@ file_cache: refresh-sec: hard-limit: true|false +# Tiered storage configuration +tiered_storage: + path: + max-size-mb: + high-threshold: <% local storage consumed which triggers eviction. Default - 80> + low-threshold: <% local storage consumed which eviction targets. Must be less than high-threshold. Default - 60> + max-eviction: + parallelism: + poll-interval-sec: + # Attribute cache related configuration attr_cache: timeout-sec: