go/filestore
1
0
Fork 0
Code Issues 1 Pull requests 1 Releases Packages Activity Actions

wip

Browse source
This commit is contained in:
Bruno Carlin 2022-07-03 20:28:50 +02:00
parent 2093344261
commit b4b3af3ffe
4 changed files with 217 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
backend.go
View file

@ -42,8 +42,14 @@ type OSBackend struct {
path string
}
func NewOSBackend(path string) *OSBackend {
return &OSBackend{
path: path,
}
}
func (ob OSBackend) Sub(path string) Backend {
return OSBackend{filepath.Join(ob.path, path)}
return &OSBackend{filepath.Join(ob.path, path)}
}
func (ob OSBackend) Create(name string) (io.WriteCloser, error) {
@ -78,6 +84,10 @@ func (ob OSBackend) Exists(name string) bool {
}
func (ob OSBackend) Delete(name string) error {
if !ob.Exists(name) {
return nil
}
if err := os.Remove(filepath.Join(ob.path, name)); err != nil {
return fmt.Errorf("cannot remove file %q: %w", name, err)
}
@ -120,6 +130,7 @@ func (mb *MemoryBackend) Open(name string) (io.ReadCloser, error) {
}
func (mb *MemoryBackend) Delete(name string) error {
delete(mb.files, name)
return nil
}

72
backend_test.go
View file

@ -2,6 +2,8 @@ package filestore
import (
"io"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
@ -10,8 +12,39 @@ import (
func TestMemoryBackend(t *testing.T) {
t.Parallel()
t.Run("it is a valid backend", func(t *testing.T) {
m := new(MemoryBackend)
ValidateBackend(t, m)
})
t.Run("read then write a file", func(t *testing.T) {
})
}
func TestOSBackend(t *testing.T) {
t.Parallel()
t.Run("it is a valid backend", func(t *testing.T) {
tmpDir := t.TempDir()
m := NewOSBackend(tmpDir)
ValidateBackend(t, m)
})
t.Run("it returns an error if the directory cannot be created", func(t *testing.T) {
assert := require.New(t)
tmpDir := t.TempDir()
fullpath := filepath.Join(tmpDir, "foo")
m := NewOSBackend(fullpath)
err := os.WriteFile(fullpath, []byte(filecontent), 0o600)
assert.NoError(err)
wc, err := m.Create("foo")
assert.Error(err)
assert.Nil(wc)
})
}
//revive:disable:cognitive-complexity
@ -20,6 +53,8 @@ func ValidateBackend(t *testing.T, b Backend) {
filecontent := []byte("file content")
setupBackend(t, b)
t.Run("validate Sub", func(t *testing.T) {
t.Run("it must not be nil", func(t *testing.T) {
s := b.Sub("foo")
@ -37,10 +72,21 @@ func ValidateBackend(t *testing.T, b Backend) {
assert.NoError(err, "Create must not return an error in case of success")
assert.NotNil(wc, "Create must return a non nil value in case of success")
_, err = wc.Write(filecontent)
assert.NoError(err)
wc.Close()
assert.True(b.Exists("create-1"), "The file must exist after "+
"Create has been successfully called")
rc, err := b.Open("create-1")
assert.NoError(err)
newcontent, err := io.ReadAll(rc)
assert.NoError(err)
assert.Equal(filecontent, newcontent, "The content of the created "+
"file must be correct")
})
t.Run("it should replace a file if it exists", func(t *testing.T) {
@ -64,6 +110,8 @@ func ValidateBackend(t *testing.T, b Backend) {
assert.NoError(err)
assert.NotEqual([]byte("hello world"), newcontent, "Create should "+
"overwrite the file when it already existed")
assert.Equal(filecontent, newcontent, "Create should "+
"overwrite the file when it already existed")
})
t.Run("when the file cannot be created", func(t *testing.T) {
@ -155,6 +203,30 @@ func ValidateBackend(t *testing.T, b Backend) {
})
}
func setupBackend(t *testing.T, b Backend) {
t.Helper()
assert := require.New(t)
wc, err := b.Create("create-2")
assert.NoError(err)
defer wc.Close()
_, err = wc.Write([]byte("hello world"))
assert.NoError(err)
wc, err = b.Create("open-1")
assert.NoError(err)
defer wc.Close()
_, err = wc.Write([]byte(filecontent))
assert.NoError(err)
wc, err = b.Create("delete-1")
assert.NoError(err)
defer wc.Close()
_, err = wc.Write([]byte(filecontent))
assert.NoError(err)
}
func precondFileExists(t *testing.T, b Backend, name string) {
t.Helper()

11
filestore.go
View file

@ -43,6 +43,13 @@ func WithNameGen(f NameGenerator) Option {
}
}
// WithBackend specify the backend the fiilestore must use.
func WithBackend(b Backend) Option {
return func(fs *Filestore) {
fs.dataDir = b
}
}
// Filestore is an abstraction of the filesystem that allow one to store and
// get files in a simple way.
type Filestore struct {
@ -52,9 +59,9 @@ type Filestore struct {
}
// NewFilestore creates a new filestore.
func NewFilestore(path string, opts ...Option) (*Filestore, error) {
func NewFilestore(opts ...Option) (*Filestore, error) {
f := &Filestore{
dataDir: OSBackend{path},
dataDir: &MemoryBackend{},
nameGenerator: NameIdentity,
buckets: map[string]*Bucket{},
}

281
filestore_test.go
View file

@ -4,8 +4,6 @@ import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"
@ -26,17 +24,7 @@ func nameGenAddress(f NameGenerator) string {
return fmt.Sprintf("%p", f)
}
func initStore(t *testing.T, path string) *Filestore {
t.Helper()
fs, err := NewFilestore(path)
require.NoError(t, err, "cannot initialize filestore")
return fs
}
func TestFilestore(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
const bucket = "foo"
@ -46,53 +34,18 @@ func TestFilestore(t *testing.T) {
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
assert.NotNil(fs)
})
t.Run("It should create its directories if they do not exist", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(filepath.Join(tmpDir, "foo", "bar", "baz"))
assert.NoError(err)
assert.NotNil(fs)
})
t.Run("Initialize a store in a directory that can't be created", func(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
tmpDir = filepath.Join(tmpDir, "unwritable")
err := os.Mkdir(tmpDir, 0o000)
assert.NoError(err)
tmpDir = filepath.Join(tmpDir, "subdir")
os.WriteFile(tmpDir, []byte("foo"), 0o644)
fs, err := NewFilestore(tmpDir)
assert.Error(err)
assert.Nil(fs)
})
t.Run("Selecting a new bucket", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
@ -105,8 +58,7 @@ func TestFilestore(t *testing.T) {
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
@ -123,13 +75,10 @@ func TestFilestore(t *testing.T) {
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
assert.NotNil(b)
assert.Equal(nameGenAddress(NameIdentity), nameGenAddress(b.nameGenerator))
assert.Equal(nameGenAddress(NameIdentity), nameGenAddress(fs.nameGenerator))
})
t.Run("A custom name generator can be used", func(t *testing.T) {
@ -141,18 +90,36 @@ func TestFilestore(t *testing.T) {
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir, WithNameGen(nameGen))
fs, err := NewFilestore(WithNameGen(nameGen))
assert.NoError(err)
b := fs.Bucket(bucket)
assert.NotNil(b)
assert.Equal(nameGenAddress(nameGen), nameGenAddress(b.nameGenerator))
assert.Equal(nameGenAddress(nameGen), nameGenAddress(fs.nameGenerator))
})
t.Run("The Default backend is memory", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
fs, err := NewFilestore()
assert.NoError(err)
assert.IsType(fs.dataDir, &MemoryBackend{})
})
t.Run("A custom backend can be used", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
fs, err := NewFilestore(WithBackend(NewOSBackend("foo")))
assert.NoError(err)
assert.IsType(fs.dataDir, &OSBackend{})
})
}
func TestBucket(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
const bucket = "foo"
@ -166,8 +133,7 @@ func TestBucket(t *testing.T) {
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
@ -177,10 +143,30 @@ func TestBucket(t *testing.T) {
assert.NotEqual(nameGenAddress(fs.nameGenerator), nameGenAddress(b.nameGenerator))
assert.Equal(nameGenAddress(nameGen), nameGenAddress(b.nameGenerator))
})
t.Run("the backend is the same as the file store", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
assert.NotNil(b)
assert.IsType(b.dataDir, fs.dataDir)
fs, err = NewFilestore(WithBackend(NewOSBackend("foo")))
assert.NoError(err)
b = fs.Bucket(bucket)
assert.NotNil(b)
assert.IsType(b.dataDir, fs.dataDir)
})
}
func TestBucketPut(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
const (
@ -190,10 +176,11 @@ func TestBucketPut(t *testing.T) {
t.Run("It should store a file", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
assert := require.New(t)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
fr := strings.NewReader(filecontent)
@ -201,9 +188,6 @@ func TestBucketPut(t *testing.T) {
name, err := b.Put(filename, fr)
assert.NoError(err, "cannot store file")
assert.Equal("foo.bar", name)
assert.FileExists(filepath.Join(tmpDir, bucket, filename),
"the file does not exist on disk")
defer b.Delete(filename)
assert.True(b.Exists(name),
"the file does not exist in the store")
@ -211,14 +195,11 @@ func TestBucketPut(t *testing.T) {
t.Run("It returns an error if the name cannot be generated", func(t *testing.T) {
t.Parallel()
errNameGen := func(_ string) (string, error) {
return "", errFilenameGeneration
}
tmpDir := t.TempDir()
assert := require.New(t)
fs, err := NewFilestore(tmpDir, WithNameGen(errNameGen))
fs, err := NewFilestore(WithNameGen(errNameGen))
assert.NoError(err)
fr := strings.NewReader(filecontent)
@ -228,48 +209,15 @@ func TestBucketPut(t *testing.T) {
assert.Equal("", name)
})
t.Run("It returns an error if the bucket dir cannot be created", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
assert := require.New(t)
fs, err := NewFilestore(tmpDir)
assert.NoError(err)
err = os.WriteFile(filepath.Join(tmpDir, bucket), []byte("foo"), 0o644)
assert.NoError(err)
err = os.Chmod(tmpDir, 0o400)
assert.NoError(err)
defer func() {
err = os.Chmod(tmpDir, 0o700)
assert.NoError(err)
}()
fr := strings.NewReader(filecontent)
name, err := fs.Bucket(bucket).Put(filename, fr)
assert.Error(err)
assert.Equal("", name)
})
t.Run("It returns an error if the file cannot be opened", func(t *testing.T) {
t.Run("It returns an error if the file cannot be stored", func(t *testing.T) {
t.Parallel()
t.Skip()
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
bucketDir := filepath.Join(tmpDir, bucket)
err = os.Mkdir(bucketDir, 0o700)
assert.NoError(err)
err = os.Mkdir(filepath.Join(bucketDir, filename), 0o700)
assert.NoError(err)
fr := strings.NewReader(filecontent)
name, err := fs.Bucket(bucket).Put(filename, fr)
@ -279,10 +227,9 @@ func TestBucketPut(t *testing.T) {
t.Run("It returns an error if the file cannot be written", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs, err := NewFilestore(tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
@ -295,7 +242,6 @@ func TestBucketPut(t *testing.T) {
}
func TestBucketGet(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
const (
@ -305,24 +251,22 @@ func TestBucketGet(t *testing.T) {
t.Run("It should retrieve a file", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
fullpath := filepath.Join(tmpDir, bucket, filename)
err := os.Mkdir(filepath.Dir(fullpath), 0o700)
r := strings.NewReader(filecontent)
_, err = b.Put(filename, r)
assert.NoError(err)
err = os.WriteFile(fullpath, []byte(filecontent), 0o600)
assert.NoError(err)
defer b.Delete(filename)
r, err := b.Get(filename)
rc, err := b.Get(filename)
assert.NoError(err)
defer func() { _ = r.Close() }()
defer func() { _ = rc.Close() }()
content, err := io.ReadAll(r)
content, err := io.ReadAll(rc)
assert.NoError(err)
assert.Equal(filecontent, string(content))
})
@ -331,8 +275,9 @@ func TestBucketGet(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
r, err := b.Get(filename)
@ -342,21 +287,14 @@ func TestBucketGet(t *testing.T) {
t.Run("It returns an error if the file cannot be read", func(t *testing.T) {
t.Parallel()
t.Skip()
assert := require.New(t)
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
fullpath := filepath.Join(tmpDir, bucket, filename)
err := os.Mkdir(filepath.Dir(fullpath), 0o700)
assert.NoError(err)
//revive:disable-next-line:add-constant // Only used here for tests
err = os.WriteFile(fullpath, []byte(filecontent), 0o100)
assert.NoError(err)
defer b.Delete(filename)
r, err := b.Get(filename)
assert.Error(err)
assert.Nil(r)
@ -364,7 +302,6 @@ func TestBucketGet(t *testing.T) {
}
func TestBucketExists(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
const (
@ -376,27 +313,24 @@ func TestBucketExists(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
fullpath := filepath.Join(tmpDir, bucket, filename)
err := os.Mkdir(filepath.Dir(fullpath), 0o700)
wc, err := b.dataDir.Create(filename)
assert.NoError(err)
err = os.WriteFile(fullpath, []byte(filecontent), 0o600)
assert.NoError(err)
defer b.Delete(filename)
wc.Close()
assert.True(b.Exists(filename))
})
t.Run("It returns false if the file does not exist", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
fs, err := NewFilestore()
assert.NoError(err)
b := fs.Bucket(bucket)
assert.False(b.Exists(filename))
@ -404,7 +338,6 @@ func TestBucketExists(t *testing.T) {
}
func TestBucketDelete(t *testing.T) {
t.Skip("not ready yet")
t.Parallel()
const (
@ -412,13 +345,13 @@ func TestBucketDelete(t *testing.T) {
filename = "foo.bar"
)
t.Run("Removing a file (happy path)", func(t *testing.T) {
t.Run("Removing a file", func(t *testing.T) {
t.Parallel()
tmpDir := t.TempDir()
fs := initStore(t, tmpDir)
assert := require.New(t)
fs, err := NewFilestore()
assert.NoError(err)
fr := strings.NewReader(filecontent)
name, err := fs.Bucket(bucket).Put(filename, fr)
@ -428,7 +361,37 @@ func TestBucketDelete(t *testing.T) {
err = fs.Bucket(bucket).Delete(name)
assert.NoError(err)
assert.NoFileExists(filepath.Join(tmpDir, bucket, filename))
assert.False(fs.Bucket("foo").Exists(name))
assert.False(fs.Bucket(bucket).Exists(name))
})
t.Run("Removing a file that do not exists", func(t *testing.T) {
t.Parallel()
assert := require.New(t)
fs, err := NewFilestore()
assert.NoError(err)
err = fs.Bucket(bucket).Delete(filename)
assert.NoError(err)
})
t.Run("Removing a file with errors", func(t *testing.T) {
t.Skip()
t.Parallel()
assert := require.New(t)
fs, err := NewFilestore()
assert.NoError(err)
err = fs.Bucket(bucket).Delete(filename)
assert.Error(err)
})
}
func TestIdentityNameGenerator(t *testing.T) {
t.Parallel()
name, err := NameIdentity("foo.txt")
require.NoError(t, err)
require.Equal(t, "foo.txt", name)
}