Skip to content

Commit cbfc8c0

Browse files
xushiweixushiwei
committed
x/http/fs/cached/remote: refactor
1 parent 646d2fb commit cbfc8c0

File tree

4 files changed

+138
-56
lines changed

4 files changed

+138
-56
lines changed

‎http/fs/cached/cached.go

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,25 +28,38 @@ import (
2828
)
2929

3030
var (
31+
// ErrOffline indicates that the remote filesystem is offline.
3132
ErrOffline = errors.New("remote filesystem is offline")
3233
)
3334

3435
const (
36+
// ModeRemote indicates that the file is a remote file or directory.
3537
ModeRemote = fs.ModeSymlink | fs.ModeIrregular
3638
)
3739

40+
// IsRemote checks if the file mode indicates a remote file or directory.
3841
func IsRemote(mode fs.FileMode) bool {
3942
return (mode & ModeRemote) == ModeRemote
4043
}
4144

4245
// -----------------------------------------------------------------------------------------
4346

47+
// Remote is an interface for remote file system operations.
4448
type Remote interface {
45-
Init(local string, offline bool)
49+
// Init initializes the remote file system with it local cache.
50+
Init(local string, offline bool) error
51+
52+
// Lstat retrieves the cached FileInfo for the specified file or directory.
4653
Lstat(localFile string) (fs.FileInfo, error)
54+
55+
// ReaddirAll reads all entries in the directory and returns their cached FileInfo.
4756
ReaddirAll(localDir string, dir *os.File, offline bool) (fis []fs.FileInfo, err error)
57+
58+
// SyncLstat retrieves the FileInfo from the remote.
4859
SyncLstat(local string, name string) (fs.FileInfo, error)
49-
SyncOpen(local string, name string) (http.File, error)
60+
61+
// SyncOpen retrieves the file from the remote.
62+
SyncOpen(local string, name string, fi fs.FileInfo) (http.File, error)
5063
}
5164

5265
type fsCached struct {
@@ -55,15 +68,31 @@ type fsCached struct {
5568
offline bool
5669
}
5770

71+
// New creates a new cached file system with the specified local cache directory
72+
// and remote file system.
5873
func New(local string, remote Remote, offline ...bool) http.FileSystem {
74+
fs, err := NewEx(local, remote, offline...)
75+
if err != nil {
76+
panic(err)
77+
}
78+
return fs
79+
}
80+
81+
// NewEx creates a new cached file system with the specified local cache directory
82+
// and remote file system.
83+
func NewEx(local string, remote Remote, offline ...bool) (_ http.FileSystem, err error) {
5984
var isOffline bool
6085
if offline != nil {
6186
isOffline = offline[0]
6287
}
63-
remote.Init(local, isOffline)
64-
return &fsCached{local, remote, isOffline}
88+
err = remote.Init(local, isOffline)
89+
if err != nil {
90+
return
91+
}
92+
return &fsCached{local, remote, isOffline}, nil
6593
}
6694

95+
// RemoteOf retrieves the remote file system from the cached file system.
6796
func RemoteOf(fs http.FileSystem) (r Remote, ok bool) {
6897
c, ok := fs.(*fsCached)
6998
if ok {
@@ -72,6 +101,7 @@ func RemoteOf(fs http.FileSystem) (r Remote, ok bool) {
72101
return
73102
}
74103

104+
// IsOffline checks if the cached file system is in offline mode.
75105
func IsOffline(fs http.FileSystem) bool {
76106
if c, ok := fs.(*fsCached); ok {
77107
return c.offline
@@ -96,7 +126,7 @@ func (p *fsCached) Open(name string) (file http.File, err error) {
96126
if p.offline {
97127
return nil, ErrOffline
98128
}
99-
return remote.SyncOpen(local, name)
129+
return remote.SyncOpen(local, name, fi)
100130
}
101131
f, err := os.Open(localFile)
102132
if err != nil {
@@ -118,6 +148,7 @@ func (p *fsCached) Open(name string) (file http.File, err error) {
118148

119149
// -----------------------------------------------------------------------------------------
120150

151+
// DownloadFile downloads the file from the remote to the local cache file.
121152
func DownloadFile(localFile string, file http.File) (err error) {
122153
_, err = file.Seek(0, io.SeekStart)
123154
if err != nil {

‎http/fs/cached/dir/dir_cache.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,11 @@ type entryHdr struct {
7777
mtime int64 // modification time in UnixMicro
7878
mode uint32 // file mode bits
7979
nameLen uint32
80+
udata uint64 // user data
8081
}
8182

8283
const (
83-
entryHdrLen = 24
84+
entryHdrLen = 32
8485
)
8586

8687
func (p *entryHdr) read(b []byte) ([]byte, error) {
@@ -91,17 +92,18 @@ func (p *entryHdr) read(b []byte) ([]byte, error) {
9192
p.mtime = int64(binary.LittleEndian.Uint64(b[8:]))
9293
p.mode = binary.LittleEndian.Uint32(b[16:])
9394
p.nameLen = binary.LittleEndian.Uint32(b[20:])
94-
return b[24:], nil
95+
p.udata = binary.LittleEndian.Uint64(b[24:])
96+
return b[32:], nil
9597
}
9698

97-
func WriteFileInfo(b []byte, fi fs.FileInfo) []byte {
99+
func WriteFileInfo(b []byte, fi fs.FileInfo, udata uint64) {
98100
binary.LittleEndian.PutUint64(b, uint64(fi.Size()))
99101
binary.LittleEndian.PutUint64(b[8:], uint64(fi.ModTime().UnixMicro()))
100102
binary.LittleEndian.PutUint32(b[16:], uint32(fi.Mode()))
101103
name := fi.Name()
102104
binary.LittleEndian.PutUint32(b[20:], uint32(len(name)))
103-
copy(b[24:], name)
104-
return b[24+len(name):]
105+
binary.LittleEndian.PutUint64(b[24:], udata)
106+
copy(b[32:], name)
105107
}
106108

107109
type fileInfo struct {
@@ -138,10 +140,14 @@ func (p *fileInfo) IsDir() bool {
138140
return p.Mode().IsDir()
139141
}
140142

141-
func (p *fileInfo) Sys() interface{} {
143+
func (p *fileInfo) Sys() any {
142144
return nil
143145
}
144146

147+
func (p *fileInfo) Udata() uint64 {
148+
return p.d.udata
149+
}
150+
145151
// -----------------------------------------------------------------------------------------
146152

147153
func ReadFileInfos(b []byte) (fis []fs.FileInfo, err error) {
@@ -174,10 +180,10 @@ func SizeFileInfos(fis []fs.FileInfo) int {
174180

175181
// -----------------------------------------------------------------------------------------
176182

177-
func BytesFileInfo(fi fs.FileInfo) []byte {
183+
func BytesFileInfo(fi fs.FileInfo, udata uint64) []byte {
178184
n := SizeFileInfo(fi)
179185
b := make([]byte, n)
180-
WriteFileInfo(b, fi)
186+
WriteFileInfo(b, fi, udata)
181187
return b
182188
}
183189

‎http/fs/cached/remote/remote.go

Lines changed: 85 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,10 @@ func SetDebug(dbgFlags int) {
4848
// -----------------------------------------------------------------------------------------
4949

5050
const (
51-
SysFilePrefix = ".fscache."
51+
SysFilePrefix = ".fscache."
52+
53+
// readDir from local is ready if there is a dirListCacheFile
54+
// see MarkDirCached
5255
dirListCacheFile = SysFilePrefix + "ls"
5356
)
5457

@@ -61,22 +64,87 @@ func checkDirCached(dir string) fs.FileInfo {
6164
return fi
6265
}
6366

64-
func TouchDirCached(dir string) error {
67+
// MarkDirCached marks the directory has dirList cache.
68+
func MarkDirCached(dir string) error {
6569
cacheFile := filepath.Join(dir, dirListCacheFile)
6670
return os.WriteFile(cacheFile, nil, 0666)
6771
}
6872

69-
func WriteStubFile(localFile string, fi fs.FileInfo) error {
73+
// WriteStubFile writes a stub file to the local file system.
74+
// If the file is a directory, it creates corresponding directory.
75+
func WriteStubFile(localFile string, fi fs.FileInfo, udata uint64) error {
7076
if fi.IsDir() {
77+
// directory don't need to save FileInfo
7178
return os.Mkdir(localFile, 0755)
7279
}
7380
fi = &fileInfoRemote{fi}
74-
b := xdir.BytesFileInfo(fi)
81+
b := xdir.BytesFileInfo(fi, udata)
7582
dest := base64.URLEncoding.EncodeToString(b)
7683
// don't need to restore mtime for symlink (saved by BytesFileInfo)
7784
return os.Symlink(dest, localFile)
7885
}
7986

87+
// Udata returns the user data from the FileInfo.
88+
func Udata(fi fs.FileInfo) uint64 {
89+
if fi, ok := fi.(interface{ Udata() uint64 }); ok {
90+
return fi.Udata()
91+
}
92+
return 0
93+
}
94+
95+
// MkStubFile creates a stub file with the specified name, size and mtime.
96+
func MkStubFile(rootDir string, name string, size int64, mtime time.Time, udata uint64) (err error) {
97+
file := filepath.Join(rootDir, name)
98+
dir, fname := filepath.Split(file)
99+
err = os.MkdirAll(dir, 0755)
100+
if err != nil {
101+
return
102+
}
103+
fi := xfs.NewFileInfo(fname, size)
104+
fi.Mtime = mtime
105+
return WriteStubFile(file, fi, udata)
106+
}
107+
108+
// ReaddirAll reads all entries in the directory and returns their cached FileInfo.
109+
func ReaddirAll(localDir string, dir *os.File, offline bool) (fis []fs.FileInfo, err error) {
110+
if fis, err = dir.Readdir(-1); err != nil {
111+
return
112+
}
113+
n := 0
114+
for _, fi := range fis {
115+
name := fi.Name()
116+
if strings.HasPrefix(name, SysFilePrefix) { // skip fscache system files
117+
continue
118+
}
119+
if isRemote(fi) {
120+
if offline {
121+
continue
122+
}
123+
localFile := filepath.Join(localDir, name)
124+
fi = readStubFile(localFile, fi)
125+
}
126+
fis[n] = fi
127+
n++
128+
}
129+
return fis[:n], nil
130+
}
131+
132+
// Lstat returns the FileInfo for the specified file or directory.
133+
func Lstat(localFile string) (fi fs.FileInfo, err error) {
134+
fi, err = os.Lstat(localFile)
135+
if err != nil {
136+
return
137+
}
138+
if fi.IsDir() {
139+
if checkDirCached(localFile) == nil { // no dir cache
140+
fi = &fileInfoRemote{fi}
141+
}
142+
} else if isRemote(fi) {
143+
fi = readStubFile(localFile, fi)
144+
}
145+
return
146+
}
147+
80148
func readStubFile(localFile string, fi fs.FileInfo) fs.FileInfo {
81149
dest, e1 := os.Readlink(localFile)
82150
if e1 == nil {
@@ -109,6 +177,7 @@ func readdir(f http.File) ([]fs.FileInfo, error) {
109177

110178
// -----------------------------------------------------------------------------------------
111179

180+
/* TODO(xsw): cache file
112181
type objFile struct {
113182
http.File
114183
localFile string
@@ -129,6 +198,7 @@ func (p *objFile) Close() error {
129198
}
130199
return file.Close()
131200
}
201+
*/
132202

133203
type fileInfoRemote struct {
134204
fs.FileInfo
@@ -147,48 +217,18 @@ type remote struct {
147217
}
148218

149219
func (p *remote) ReaddirAll(localDir string, dir *os.File, offline bool) (fis []fs.FileInfo, err error) {
150-
if fis, err = dir.Readdir(-1); err != nil {
151-
return
152-
}
153-
n := 0
154-
for _, fi := range fis {
155-
name := fi.Name()
156-
if strings.HasPrefix(name, SysFilePrefix) { // skip fscache system files
157-
continue
158-
}
159-
if isRemote(fi) {
160-
if offline {
161-
continue
162-
}
163-
localFile := filepath.Join(localDir, name)
164-
fi = readStubFile(localFile, fi)
165-
}
166-
fis[n] = fi
167-
n++
168-
}
169-
return fis[:n], nil
220+
return ReaddirAll(localDir, dir, offline)
170221
}
171222

172223
func (p *remote) Lstat(localFile string) (fi fs.FileInfo, err error) {
173-
fi, err = os.Lstat(localFile)
174-
if err != nil {
175-
return
176-
}
177-
if fi.IsDir() {
178-
if checkDirCached(localFile) == nil { // no dir cache
179-
fi = &fileInfoRemote{fi}
180-
}
181-
} else if isRemote(fi) {
182-
fi = readStubFile(localFile, fi)
183-
}
184-
return
224+
return Lstat(localFile)
185225
}
186226

187227
func (p *remote) SyncLstat(local string, name string) (fi fs.FileInfo, err error) {
188228
return nil, os.ErrNotExist
189229
}
190230

191-
func (p *remote) SyncOpen(local string, name string) (f http.File, err error) {
231+
func (p *remote) SyncOpen(local string, name string, fi fs.FileInfo) (f http.File, err error) {
192232
f, err = p.bucket.Open(name)
193233
if err != nil {
194234
log.Printf(`[ERROR] bucket.Open("%s"): %v\n`, name, err)
@@ -197,7 +237,7 @@ func (p *remote) SyncOpen(local string, name string) (f http.File, err error) {
197237
if debugNet {
198238
log.Println("[INFO] ==> bucket.Open", name)
199239
}
200-
if f.(interface{ IsDir() bool }).IsDir() {
240+
if fi.IsDir() {
201241
fis, e := readdir(f)
202242
if e != nil {
203243
log.Printf(`[ERROR] Readdir("%s"): %v\n`, name, e)
@@ -211,26 +251,29 @@ func (p *remote) SyncOpen(local string, name string) (f http.File, err error) {
211251
base := filepath.Join(local, name)
212252
for _, fi := range fis {
213253
itemFile := base + "/" + fi.Name()
214-
if WriteStubFile(itemFile, fi) != nil {
254+
if WriteStubFile(itemFile, fi, 0) != nil {
215255
nError++
216256
}
217257
}
218258
if nError == 0 {
219-
TouchDirCached(base)
259+
MarkDirCached(base)
220260
} else {
221261
log.Printf("[WARN] writeStubFile fail (%d errors)", nError)
222262
}
223263
}()
224264
return xfs.Dir(f, fis), nil
225265
}
266+
/* TODO(xsw):
226267
if p.cacheFile {
227268
localFile := filepath.Join(local, name)
228269
f = &objFile{f, localFile, p.notify}
229270
}
271+
*/
230272
return
231273
}
232274

233-
func (p *remote) Init(local string, offline bool) {
275+
func (p *remote) Init(local string, offline bool) error {
276+
return nil
234277
}
235278

236279
// -----------------------------------------------------------------------------------------
@@ -239,6 +282,7 @@ type NotifyFile interface {
239282
NotifyFile(ctx context.Context, name string, fi fs.FileInfo)
240283
}
241284

285+
// NewRemote creates a new remote file system.
242286
func NewRemote(fsRemote http.FileSystem, notifyOrNil NotifyFile, cacheFile bool) (ret cached.Remote, err error) {
243287
return &remote{fsRemote, notifyOrNil, cacheFile}, nil
244288
}

0 commit comments

Comments
 (0)