add webdav depth=1

package.json

@@ -54,6 +54,7 @@
     "@aws-sdk/lib-storage": "^3.40.1",
     "@aws-sdk/signature-v4-crt": "^3.37.0",
     "@azure/msal-node": "^1.4.0",
+    "@fyears/tsqueue": "^1.0.1",
     "@microsoft/microsoft-graph-client": "^3.0.1",
     "acorn": "^8.5.0",
     "assert": "^2.0.0",

src/baseTypes.ts

@@ -31,6 +31,7 @@ export interface WebdavConfig {
   username: string;
   password: string;
   authType: WebdavAuthType;
+  manualRecursive: boolean;
 }
 
 export interface OnedriveConfig {

src/main.ts

@@ -141,63 +141,63 @@ export default class RemotelySavePlugin extends Plugin {
         () => self.saveSettings()
       );
       const remoteRsp = await client.listFromRemote();
-      // log.info(remoteRsp);
+      log.info(remoteRsp);
 
-      getNotice("3/7 Starting to fetch local meta data.");
+      // getNotice("3/7 Starting to fetch local meta data.");
-      this.syncStatus = "getting_local_meta";
+      // this.syncStatus = "getting_local_meta";
-      const local = this.app.vault.getAllLoadedFiles();
+      // const local = this.app.vault.getAllLoadedFiles();
-      const localHistory = await loadDeleteRenameHistoryTableByVault(
+      // const localHistory = await loadDeleteRenameHistoryTableByVault(
-        this.db,
+      //   this.db,
-        this.settings.vaultRandomID
+      //   this.settings.vaultRandomID
-      );
+      // );
-      // log.info(local);
+      // // log.info(local);
-      // log.info(localHistory);
+      // // log.info(localHistory);
 
-      getNotice("4/7 Checking password correct or not.");
+      // getNotice("4/7 Checking password correct or not.");
-      this.syncStatus = "checking_password";
+      // this.syncStatus = "checking_password";
-      const passwordCheckResult = await isPasswordOk(
+      // const passwordCheckResult = await isPasswordOk(
-        remoteRsp.Contents,
+      //   remoteRsp.Contents,
-        this.settings.password
+      //   this.settings.password
-      );
+      // );
-      if (!passwordCheckResult.ok) {
+      // if (!passwordCheckResult.ok) {
-        getNotice("something goes wrong while checking password");
+      //   getNotice("something goes wrong while checking password");
-        throw Error(passwordCheckResult.reason);
+      //   throw Error(passwordCheckResult.reason);
-      }
+      // }
 
-      getNotice("5/7 Starting to generate sync plan.");
+      // getNotice("5/7 Starting to generate sync plan.");
-      this.syncStatus = "generating_plan";
+      // this.syncStatus = "generating_plan";
-      const syncPlan = await getSyncPlan(
+      // const syncPlan = await getSyncPlan(
-        remoteRsp.Contents,
+      //   remoteRsp.Contents,
-        local,
+      //   local,
-        localHistory,
+      //   localHistory,
-        this.db,
+      //   this.db,
-        this.settings.vaultRandomID,
+      //   this.settings.vaultRandomID,
-        client.serviceType,
+      //   client.serviceType,
-        this.settings.password
+      //   this.settings.password
-      );
+      // );
-      log.info(syncPlan.mixedStates); // for debugging
+      // log.info(syncPlan.mixedStates); // for debugging
-      await insertSyncPlanRecordByVault(
+      // await insertSyncPlanRecordByVault(
-        this.db,
+      //   this.db,
-        syncPlan,
+      //   syncPlan,
-        this.settings.vaultRandomID
+      //   this.settings.vaultRandomID
-      );
+      // );
 
-      // The operations above are read only and kind of safe.
+      // // The operations above are read only and kind of safe.
-      // The operations below begins to write or delete (!!!) something.
+      // // The operations below begins to write or delete (!!!) something.
 
-      getNotice("6/7 Remotely Save Sync data exchanging!");
+      // getNotice("6/7 Remotely Save Sync data exchanging!");
 
-      this.syncStatus = "syncing";
+      // this.syncStatus = "syncing";
-      await doActualSync(
+      // await doActualSync(
-        client,
+      //   client,
-        this.db,
+      //   this.db,
-        this.settings.vaultRandomID,
+      //   this.settings.vaultRandomID,
-        this.app.vault,
+      //   this.app.vault,
-        syncPlan,
+      //   syncPlan,
-        this.settings.password,
+      //   this.settings.password,
-        (i: number, totalCount: number, pathName: string, decision: string) =>
+      //   (i: number, totalCount: number, pathName: string, decision: string) =>
-          self.setCurrSyncMsg(i, totalCount, pathName, decision)
+      //     self.setCurrSyncMsg(i, totalCount, pathName, decision)
-      );
+      // );
 
       getNotice("7/7 Remotely Save finish!");
       this.currSyncMsg = "";
@@ -504,6 +504,9 @@ export default class RemotelySavePlugin extends Plugin {
     if (this.settings.onedrive.authority === "") {
       this.settings.onedrive.authority = DEFAULT_SETTINGS.onedrive.authority;
     }
+    if (this.settings.webdav.manualRecursive === undefined) {
+      this.settings.webdav.manualRecursive = false;
+    }
   }
 
   async saveSettings() {

src/remoteForWebdav.ts

@@ -2,6 +2,9 @@ import { Buffer } from "buffer";
 import { Vault } from "obsidian";
 import type { FileStat, WebDAVClient } from "webdav/web";
 import { AuthType, BufferLike, createClient } from "webdav/web";
+import { Queue } from "@fyears/tsqueue";
+import chunk from "lodash/chunk";
+import flatten from "lodash/flatten";
 import type { RemoteItem, WebdavConfig } from "./baseTypes";
 import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
 import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
@@ -15,6 +18,7 @@ export const DEFAULT_WEBDAV_CONFIG = {
   username: "",
   password: "",
   authType: "basic",
+  manualRecursive: false,
 } as WebdavConfig;
 
 const getWebdavPath = (fileOrFolderPath: string, vaultName: string) => {
@@ -199,7 +203,43 @@ export const listFromRemote = async (
     throw Error("prefix not supported");
   }
   await client.init();
-  const contents = (await client.client.getDirectoryContents(
+
+  let contents = [] as FileStat[];
+  if (client.webdavConfig.manualRecursive) {
+    // the remote doesn't support infinity propfind,
+    // we need to do a bfs here
+    const q = new Queue([`/${client.vaultName}`]);
+    const CHUNK_SIZE = 10;
+    while (q.length > 0) {
+      const itemsToFetch = [];
+      while (q.length > 0) {
+        itemsToFetch.push(q.pop());
+      }
+      const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE);
+      // log.debug(itemsToFetchChunks);
+      const subContents = [] as FileStat[];
+      for (const singleChunk of itemsToFetchChunks) {
+        const r = singleChunk.map((x) => {
+          return client.client.getDirectoryContents(x, {
+            deep: false,
+            details: false /* no need for verbose details here */,
+            glob: "/**" /* avoid dot files by using glob */,
+          }) as Promise<FileStat[]>;
+        });
+        const r2 = flatten(await Promise.all(r));
+        subContents.push(...r2);
+      }
+      for (let i = 0; i < subContents.length; ++i) {
+        const f = subContents[i];
+        contents.push(f);
+        if (f.type === "directory") {
+          q.push(f.filename);
+        }
+      }
+    }
+  } else {
+    // the remote supports infinity propfind
+    contents = (await client.client.getDirectoryContents(
       `/${client.vaultName}`,
       {
         deep: true,
@@ -207,6 +247,7 @@ export const listFromRemote = async (
         glob: "/**" /* avoid dot files by using glob */,
       }
     )) as FileStat[];
+  }
   return {
     Contents: contents.map((x) =>
       fromWebdavItemToRemoteItem(x, client.vaultName)

src/settings.ts

@@ -872,6 +872,32 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    new Setting(webdavDiv)
+      .setName("server supports infinity propfind or not")
+      .setDesc(
+        "The plugin needs to get all files and folders recursively using probfind. If your webdav server only supports depth='1' (such as NGINX), you need to adjust the setting here, then the plugin consumes more network requests, but better than not working."
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("infinity", "supports depth='infinity'");
+        dropdown.addOption("1", "only supports depth='1'");
+
+        type Depth = "1" | "infinity";
+        dropdown
+          .setValue(
+            this.plugin.settings.webdav.manualRecursive === false
+              ? "infinity"
+              : "1"
+          )
+          .onChange(async (val: Depth) => {
+            if (val === "1") {
+              this.plugin.settings.webdav.manualRecursive = true;
+            } else if (val === "infinity") {
+              this.plugin.settings.webdav.manualRecursive = false;
+            }
+            await this.plugin.saveSettings();
+          });
+      });
+
     new Setting(webdavDiv)
       .setName("check connectivity")
       .setDesc("check connectivity")