From 63c54d19567ad9ef1cacaf59c0cef40ab52815eb Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 18 May 2024 01:33:20 +0800 Subject: [PATCH] check password using walkPartial instead of cache --- src/fsAll.ts | 1 + src/fsDropbox.ts | 40 +++++++++++++++++++++++++--------------- src/fsEncrypt.ts | 32 ++++++++++---------------------- src/fsLocal.ts | 4 ++++ src/fsMock.ts | 4 ++++ src/fsOnedrive.ts | 25 +++++++++++++++++++++++++ src/fsS3.ts | 43 ++++++++++++++++++++++++++++++------------- src/fsWebdav.ts | 13 +++++++++++++ src/fsWebdis.ts | 19 +++++++++++++++++++ 9 files changed, 131 insertions(+), 50 deletions(-) diff --git a/src/fsAll.ts b/src/fsAll.ts index e995bce..f1480b2 100644 --- a/src/fsAll.ts +++ b/src/fsAll.ts @@ -3,6 +3,7 @@ import type { Entity } from "./baseTypes"; export abstract class FakeFs { abstract kind: string; abstract walk(): Promise; + abstract walkPartial(): Promise; abstract stat(key: string): Promise; abstract mkdir(key: string, mtime?: number, ctime?: number): Promise; abstract writeFile( diff --git a/src/fsDropbox.ts b/src/fsDropbox.ts index de6b590..ea4db91 100644 --- a/src/fsDropbox.ts +++ b/src/fsDropbox.ts @@ -452,13 +452,21 @@ export class FakeFsDropbox extends FakeFs { } async walk(): Promise { + return await this._walk(false); + } + + async walkPartial(): Promise { + return await this._walk(true); + } + + async _walk(partial: boolean): Promise { await this._init(); let res = await this.dropbox.filesListFolder({ path: `/${this.remoteBaseDir}`, - recursive: true, + recursive: !partial, include_deleted: false, - limit: 1000, + limit: partial ? 10 : 1000, }); if (res.status !== 200) { throw Error(JSON.stringify(res)); @@ -471,20 +479,22 @@ export class FakeFsDropbox extends FakeFs { .filter((x) => x.path_display !== `/${this.remoteBaseDir}`) .map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir)); - while (res.result.has_more) { - res = await this.dropbox.filesListFolderContinue({ - cursor: res.result.cursor, - }); - if (res.status !== 200) { - throw Error(JSON.stringify(res)); - } + if (!partial) { + while (res.result.has_more) { + res = await this.dropbox.filesListFolderContinue({ + cursor: res.result.cursor, + }); + if (res.status !== 200) { + throw Error(JSON.stringify(res)); + } - const contents2 = res.result.entries; - const unifiedContents2 = contents2 - .filter((x) => x[".tag"] !== "deleted") - .filter((x) => x.path_display !== `/${this.remoteBaseDir}`) - .map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir)); - unifiedContents.push(...unifiedContents2); + const contents2 = res.result.entries; + const unifiedContents2 = contents2 + .filter((x) => x[".tag"] !== "deleted") + .filter((x) => x.path_display !== `/${this.remoteBaseDir}`) + .map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir)); + unifiedContents.push(...unifiedContents2); + } } fixEntityListCasesInplace(unifiedContents); diff --git a/src/fsEncrypt.ts b/src/fsEncrypt.ts index 648b443..2cdb911 100644 --- a/src/fsEncrypt.ts +++ b/src/fsEncrypt.ts @@ -78,8 +78,6 @@ export class FakeFsEncrypt extends FakeFs { cacheMapOrigToEnc: Record; hasCacheMap: boolean; kind: string; - innerWalkResultCache?: Entity[]; - innerWalkResultCacheTime?: number; constructor(innerFs: FakeFs, password: string, method: CipherMethodType) { super(); @@ -110,26 +108,8 @@ export class FakeFsEncrypt extends FakeFs { throw Error(`no idea about isFolderAware for method=${this.method}`); } - /** - * we want a little caching here. - */ - async _getInnerWalkResult(): Promise { - let innerWalkResult: Entity[] | undefined = undefined; - if ( - this.innerWalkResultCacheTime !== undefined && - this.innerWalkResultCacheTime >= Date.now() - 1000 - ) { - innerWalkResult = this.innerWalkResultCache!; - } else { - innerWalkResult = await this.innerFs.walk(); - this.innerWalkResultCache = innerWalkResult; - this.innerWalkResultCacheTime = Date.now(); - } - return innerWalkResult; - } - async isPasswordOk(): Promise { - const innerWalkResult = await this._getInnerWalkResult(); + const innerWalkResult = await this.walkPartial(); if (innerWalkResult === undefined || innerWalkResult.length === 0) { // remote empty @@ -186,8 +166,16 @@ export class FakeFsEncrypt extends FakeFs { } async walk(): Promise { - const innerWalkResult = await this._getInnerWalkResult(); + const innerWalkResult = await this.innerFs.walk(); + return await this._dealWithWalk(innerWalkResult); + } + async walkPartial(): Promise { + const innerWalkResult = await this.innerFs.walkPartial(); + return await this._dealWithWalk(innerWalkResult); + } + + async _dealWithWalk(innerWalkResult: Entity[]): Promise { const res: Entity[] = []; if (this.isPasswordEmpty()) { diff --git a/src/fsLocal.ts b/src/fsLocal.ts index fe101aa..4ca1bfc 100644 --- a/src/fsLocal.ts +++ b/src/fsLocal.ts @@ -109,6 +109,10 @@ export class FakeFsLocal extends FakeFs { return local; } + async walkPartial(): Promise { + return await this.walk(); + } + async stat(key: string): Promise { const statRes = await statFix(this.vault, key); if (statRes === undefined || statRes === null) { diff --git a/src/fsMock.ts b/src/fsMock.ts index 4f9d71c..4925c93 100644 --- a/src/fsMock.ts +++ b/src/fsMock.ts @@ -13,6 +13,10 @@ export class FakeFsMock extends FakeFs { throw new Error("Method not implemented."); } + async walkPartial(): Promise { + return await this.walk(); + } + async stat(key: string): Promise { throw new Error("Method not implemented."); } diff --git a/src/fsOnedrive.ts b/src/fsOnedrive.ts index 885cdf3..8bdb999 100644 --- a/src/fsOnedrive.ts +++ b/src/fsOnedrive.ts @@ -682,6 +682,31 @@ export class FakeFsOnedrive extends FakeFs { return unifiedContents; } + async walkPartial(): Promise { + await this._init(); + + const DELTA_LINK_KEY = "@odata.deltaLink"; + + const res = await this._getJson( + `/drive/special/approot:/${this.remoteBaseDir}:/delta` + ); + const driveItems = res.value as DriveItem[]; + // console.debug(driveItems); + + // lastly we should have delta link? + if (DELTA_LINK_KEY in res) { + this.onedriveConfig.deltaLink = res[DELTA_LINK_KEY]; + await this.saveUpdatedConfigFunc(); + } + + // unify everything to Entity + const unifiedContents = driveItems + .map((x) => fromDriveItemToEntity(x, this.remoteBaseDir)) + .filter((x) => x.key !== "/"); + + return unifiedContents; + } + async stat(key: string): Promise { await this._init(); return await this._statFromRoot(getOnedrivePath(key, this.remoteBaseDir)); diff --git a/src/fsS3.ts b/src/fsS3.ts index d5bf065..280b02d 100644 --- a/src/fsS3.ts +++ b/src/fsS3.ts @@ -399,9 +399,16 @@ export class FakeFsS3 extends FakeFs { } async walk(): Promise { - const res = (await this._walkFromRoot(this.s3Config.remotePrefix)).filter( - (x) => x.key !== "" && x.key !== "/" - ); + const res = ( + await this._walkFromRoot(this.s3Config.remotePrefix, false) + ).filter((x) => x.key !== "" && x.key !== "/"); + return res; + } + + async walkPartial(): Promise { + const res = ( + await this._walkFromRoot(this.s3Config.remotePrefix, true) + ).filter((x) => x.key !== "" && x.key !== "/"); return res; } @@ -409,19 +416,23 @@ export class FakeFsS3 extends FakeFs { * the input key contains basedir (prefix), * but the result doesn't contain it. */ - async _walkFromRoot(prefixOfRawKeys: string | undefined) { + async _walkFromRoot(prefixOfRawKeys: string | undefined, partial: boolean) { const confCmd = { Bucket: this.s3Config.s3BucketName, } as ListObjectsV2CommandInput; if (prefixOfRawKeys !== undefined && prefixOfRawKeys !== "") { confCmd.Prefix = prefixOfRawKeys; } + if (partial) { + confCmd.MaxKeys = 10; // no need to list more! + } const contents = [] as _Object[]; const mtimeRecords: Record = {}; const ctimeRecords: Record = {}; + const partsConcurrency = partial ? 1 : this.s3Config.partsConcurrency; const queueHead = new PQueue({ - concurrency: this.s3Config.partsConcurrency, + concurrency: partsConcurrency, autoStart: true, }); queueHead.on("error", (error) => { @@ -473,14 +484,20 @@ export class FakeFsS3 extends FakeFs { } } - isTruncated = rsp.IsTruncated ?? false; - confCmd.ContinuationToken = rsp.NextContinuationToken; - if ( - isTruncated && - (confCmd.ContinuationToken === undefined || - confCmd.ContinuationToken === "") - ) { - throw Error("isTruncated is true but no continuationToken provided"); + if (partial) { + // do not loop over + isTruncated = false; + } else { + // loop over + isTruncated = rsp.IsTruncated ?? false; + confCmd.ContinuationToken = rsp.NextContinuationToken; + if ( + isTruncated && + (confCmd.ContinuationToken === undefined || + confCmd.ContinuationToken === "") + ) { + throw Error("isTruncated is true but no continuationToken provided"); + } } } while (isTruncated); diff --git a/src/fsWebdav.ts b/src/fsWebdav.ts index 298ce02..683a1f5 100644 --- a/src/fsWebdav.ts +++ b/src/fsWebdav.ts @@ -357,6 +357,19 @@ export class FakeFsWebdav extends FakeFs { return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir)); } + async walkPartial(): Promise { + await this._init(); + + const contents = (await this.client.getDirectoryContents( + `/${this.remoteBaseDir}`, + { + deep: false, // partial, no need to recursive here + details: false /* no need for verbose details here */, + } + )) as FileStat[]; + return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir)); + } + async stat(key: string): Promise { await this._init(); const fullPath = getWebdavPath(key, this.remoteBaseDir); diff --git a/src/fsWebdis.ts b/src/fsWebdis.ts index 6ed216e..5d89cb6 100644 --- a/src/fsWebdis.ts +++ b/src/fsWebdis.ts @@ -133,6 +133,25 @@ export class FakeFsWebdis extends FakeFs { return res; } + async walkPartial(): Promise { + let cursor = "0"; + const res: Entity[] = []; + const command = `SCAN/${cursor}/MATCH/rs:fs:v1:*:meta/COUNT/10`; // fewer keys + const rsp = (await (await this._fetchCommand("GET", command)).json())[ + "SCAN" + ]; + // console.debug(rsp); + cursor = rsp[0]; + for (const fullKeyWithMeta of rsp[1]) { + const realKey = getOrigPath(fullKeyWithMeta, this.remoteBaseDir); + res.push(await this.stat(realKey)); + } + // no need to loop over cursor + // console.debug(`walk res:`); + // console.debug(res); + return res; + } + async stat(key: string): Promise { const fullKey = getWebdisPath(key, this.remoteBaseDir); return await this._statFromRaw(fullKey);