check password using walkPartial instead of cache

src/fsAll.ts

@@ -3,6 +3,7 @@ import type { Entity } from "./baseTypes";
 export abstract class FakeFs {
   abstract kind: string;
   abstract walk(): Promise<Entity[]>;
+  abstract walkPartial(): Promise<Entity[]>;
   abstract stat(key: string): Promise<Entity>;
   abstract mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity>;
   abstract writeFile(

src/fsDropbox.ts

@@ -452,13 +452,21 @@ export class FakeFsDropbox extends FakeFs {
   }
 
   async walk(): Promise<Entity[]> {
+    return await this._walk(false);
+  }
+
+  async walkPartial(): Promise<Entity[]> {
+    return await this._walk(true);
+  }
+
+  async _walk(partial: boolean): Promise<Entity[]> {
     await this._init();
 
     let res = await this.dropbox.filesListFolder({
       path: `/${this.remoteBaseDir}`,
-      recursive: true,
+      recursive: !partial,
       include_deleted: false,
-      limit: 1000,
+      limit: partial ? 10 : 1000,
     });
     if (res.status !== 200) {
       throw Error(JSON.stringify(res));
@@ -471,20 +479,22 @@ export class FakeFsDropbox extends FakeFs {
       .filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
       .map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
 
-    while (res.result.has_more) {
-      res = await this.dropbox.filesListFolderContinue({
-        cursor: res.result.cursor,
-      });
-      if (res.status !== 200) {
-        throw Error(JSON.stringify(res));
-      }
+    if (!partial) {
+      while (res.result.has_more) {
+        res = await this.dropbox.filesListFolderContinue({
+          cursor: res.result.cursor,
+        });
+        if (res.status !== 200) {
+          throw Error(JSON.stringify(res));
+        }
 
-      const contents2 = res.result.entries;
-      const unifiedContents2 = contents2
-        .filter((x) => x[".tag"] !== "deleted")
-        .filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
-        .map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
-      unifiedContents.push(...unifiedContents2);
+        const contents2 = res.result.entries;
+        const unifiedContents2 = contents2
+          .filter((x) => x[".tag"] !== "deleted")
+          .filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
+          .map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
+        unifiedContents.push(...unifiedContents2);
+      }
     }
 
     fixEntityListCasesInplace(unifiedContents);

src/fsEncrypt.ts

@@ -78,8 +78,6 @@ export class FakeFsEncrypt extends FakeFs {
   cacheMapOrigToEnc: Record<string, string>;
   hasCacheMap: boolean;
   kind: string;
-  innerWalkResultCache?: Entity[];
-  innerWalkResultCacheTime?: number;
 
   constructor(innerFs: FakeFs, password: string, method: CipherMethodType) {
     super();
@@ -110,26 +108,8 @@ export class FakeFsEncrypt extends FakeFs {
     throw Error(`no idea about isFolderAware for method=${this.method}`);
   }
 
-  /**
-   * we want a little caching here.
-   */
-  async _getInnerWalkResult(): Promise<Entity[]> {
-    let innerWalkResult: Entity[] | undefined = undefined;
-    if (
-      this.innerWalkResultCacheTime !== undefined &&
-      this.innerWalkResultCacheTime >= Date.now() - 1000
-    ) {
-      innerWalkResult = this.innerWalkResultCache!;
-    } else {
-      innerWalkResult = await this.innerFs.walk();
-      this.innerWalkResultCache = innerWalkResult;
-      this.innerWalkResultCacheTime = Date.now();
-    }
-    return innerWalkResult;
-  }
-
   async isPasswordOk(): Promise<PasswordCheckType> {
-    const innerWalkResult = await this._getInnerWalkResult();
+    const innerWalkResult = await this.walkPartial();
 
     if (innerWalkResult === undefined || innerWalkResult.length === 0) {
       // remote empty
@@ -186,8 +166,16 @@ export class FakeFsEncrypt extends FakeFs {
   }
 
   async walk(): Promise<Entity[]> {
-    const innerWalkResult = await this._getInnerWalkResult();
+    const innerWalkResult = await this.innerFs.walk();
+    return await this._dealWithWalk(innerWalkResult);
+  }
 
+  async walkPartial(): Promise<Entity[]> {
+    const innerWalkResult = await this.innerFs.walkPartial();
+    return await this._dealWithWalk(innerWalkResult);
+  }
+
+  async _dealWithWalk(innerWalkResult: Entity[]): Promise<Entity[]> {
     const res: Entity[] = [];
 
     if (this.isPasswordEmpty()) {

src/fsLocal.ts

@@ -109,6 +109,10 @@ export class FakeFsLocal extends FakeFs {
     return local;
   }
 
+  async walkPartial(): Promise<Entity[]> {
+    return await this.walk();
+  }
+
   async stat(key: string): Promise<Entity> {
     const statRes = await statFix(this.vault, key);
     if (statRes === undefined || statRes === null) {

src/fsMock.ts

@@ -13,6 +13,10 @@ export class FakeFsMock extends FakeFs {
     throw new Error("Method not implemented.");
   }
 
+  async walkPartial(): Promise<Entity[]> {
+    return await this.walk();
+  }
+
   async stat(key: string): Promise<Entity> {
     throw new Error("Method not implemented.");
   }

src/fsOnedrive.ts

@@ -682,6 +682,31 @@ export class FakeFsOnedrive extends FakeFs {
     return unifiedContents;
   }
 
+  async walkPartial(): Promise<Entity[]> {
+    await this._init();
+
+    const DELTA_LINK_KEY = "@odata.deltaLink";
+
+    const res = await this._getJson(
+      `/drive/special/approot:/${this.remoteBaseDir}:/delta`
+    );
+    const driveItems = res.value as DriveItem[];
+    // console.debug(driveItems);
+
+    // lastly we should have delta link?
+    if (DELTA_LINK_KEY in res) {
+      this.onedriveConfig.deltaLink = res[DELTA_LINK_KEY];
+      await this.saveUpdatedConfigFunc();
+    }
+
+    // unify everything to Entity
+    const unifiedContents = driveItems
+      .map((x) => fromDriveItemToEntity(x, this.remoteBaseDir))
+      .filter((x) => x.key !== "/");
+
+    return unifiedContents;
+  }
+
   async stat(key: string): Promise<Entity> {
     await this._init();
     return await this._statFromRoot(getOnedrivePath(key, this.remoteBaseDir));

src/fsS3.ts

@@ -399,9 +399,16 @@ export class FakeFsS3 extends FakeFs {
   }
 
   async walk(): Promise<Entity[]> {
-    const res = (await this._walkFromRoot(this.s3Config.remotePrefix)).filter(
-      (x) => x.key !== "" && x.key !== "/"
-    );
+    const res = (
+      await this._walkFromRoot(this.s3Config.remotePrefix, false)
+    ).filter((x) => x.key !== "" && x.key !== "/");
+    return res;
+  }
+
+  async walkPartial(): Promise<Entity[]> {
+    const res = (
+      await this._walkFromRoot(this.s3Config.remotePrefix, true)
+    ).filter((x) => x.key !== "" && x.key !== "/");
     return res;
   }
 
@@ -409,19 +416,23 @@ export class FakeFsS3 extends FakeFs {
    * the input key contains basedir (prefix),
    * but the result doesn't contain it.
    */
-  async _walkFromRoot(prefixOfRawKeys: string | undefined) {
+  async _walkFromRoot(prefixOfRawKeys: string | undefined, partial: boolean) {
     const confCmd = {
       Bucket: this.s3Config.s3BucketName,
     } as ListObjectsV2CommandInput;
     if (prefixOfRawKeys !== undefined && prefixOfRawKeys !== "") {
       confCmd.Prefix = prefixOfRawKeys;
     }
+    if (partial) {
+      confCmd.MaxKeys = 10; // no need to list more!
+    }
 
     const contents = [] as _Object[];
     const mtimeRecords: Record<string, number> = {};
     const ctimeRecords: Record<string, number> = {};
+    const partsConcurrency = partial ? 1 : this.s3Config.partsConcurrency;
     const queueHead = new PQueue({
-      concurrency: this.s3Config.partsConcurrency,
+      concurrency: partsConcurrency,
       autoStart: true,
     });
     queueHead.on("error", (error) => {
@@ -473,14 +484,20 @@ export class FakeFsS3 extends FakeFs {
         }
       }
 
-      isTruncated = rsp.IsTruncated ?? false;
-      confCmd.ContinuationToken = rsp.NextContinuationToken;
-      if (
-        isTruncated &&
-        (confCmd.ContinuationToken === undefined ||
-          confCmd.ContinuationToken === "")
-      ) {
-        throw Error("isTruncated is true but no continuationToken provided");
+      if (partial) {
+        // do not loop over
+        isTruncated = false;
+      } else {
+        // loop over
+        isTruncated = rsp.IsTruncated ?? false;
+        confCmd.ContinuationToken = rsp.NextContinuationToken;
+        if (
+          isTruncated &&
+          (confCmd.ContinuationToken === undefined ||
+            confCmd.ContinuationToken === "")
+        ) {
+          throw Error("isTruncated is true but no continuationToken provided");
+        }
       }
     } while (isTruncated);
 

src/fsWebdav.ts

@@ -357,6 +357,19 @@ export class FakeFsWebdav extends FakeFs {
     return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir));
   }
 
+  async walkPartial(): Promise<Entity[]> {
+    await this._init();
+
+    const contents = (await this.client.getDirectoryContents(
+      `/${this.remoteBaseDir}`,
+      {
+        deep: false, // partial, no need to recursive here
+        details: false /* no need for verbose details here */,
+      }
+    )) as FileStat[];
+    return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir));
+  }
+
   async stat(key: string): Promise<Entity> {
     await this._init();
     const fullPath = getWebdavPath(key, this.remoteBaseDir);

src/fsWebdis.ts

@@ -133,6 +133,25 @@ export class FakeFsWebdis extends FakeFs {
     return res;
   }
 
+  async walkPartial(): Promise<Entity[]> {
+    let cursor = "0";
+    const res: Entity[] = [];
+    const command = `SCAN/${cursor}/MATCH/rs:fs:v1:*:meta/COUNT/10`; // fewer keys
+    const rsp = (await (await this._fetchCommand("GET", command)).json())[
+      "SCAN"
+    ];
+    // console.debug(rsp);
+    cursor = rsp[0];
+    for (const fullKeyWithMeta of rsp[1]) {
+      const realKey = getOrigPath(fullKeyWithMeta, this.remoteBaseDir);
+      res.push(await this.stat(realKey));
+    }
+    // no need to loop over cursor
+    // console.debug(`walk res:`);
+    // console.debug(res);
+    return res;
+  }
+
   async stat(key: string): Promise<Entity> {
     const fullKey = getWebdisPath(key, this.remoteBaseDir);
     return await this._statFromRaw(fullKey);