From 320f91f1a50a81619c01e2ba65820c494907aa5d Mon Sep 17 00:00:00 2001 From: fyears <1142836+fyears@users.noreply.github.com> Date: Sat, 22 Jan 2022 17:11:12 +0800 Subject: [PATCH] add webdav depth=1 --- package.json | 1 + src/baseTypes.ts | 1 + src/main.ts | 105 +++++++++++++++++++++-------------------- src/remoteForWebdav.ts | 55 ++++++++++++++++++--- src/settings.ts | 26 ++++++++++ 5 files changed, 130 insertions(+), 58 deletions(-) diff --git a/package.json b/package.json index 0c024cc..bdff84b 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "@aws-sdk/lib-storage": "^3.40.1", "@aws-sdk/signature-v4-crt": "^3.37.0", "@azure/msal-node": "^1.4.0", + "@fyears/tsqueue": "^1.0.1", "@microsoft/microsoft-graph-client": "^3.0.1", "acorn": "^8.5.0", "assert": "^2.0.0", diff --git a/src/baseTypes.ts b/src/baseTypes.ts index c16bd54..946744b 100644 --- a/src/baseTypes.ts +++ b/src/baseTypes.ts @@ -31,6 +31,7 @@ export interface WebdavConfig { username: string; password: string; authType: WebdavAuthType; + manualRecursive: boolean; } export interface OnedriveConfig { diff --git a/src/main.ts b/src/main.ts index 64b8ed1..a8dbb0b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -141,63 +141,63 @@ export default class RemotelySavePlugin extends Plugin { () => self.saveSettings() ); const remoteRsp = await client.listFromRemote(); - // log.info(remoteRsp); + log.info(remoteRsp); - getNotice("3/7 Starting to fetch local meta data."); - this.syncStatus = "getting_local_meta"; - const local = this.app.vault.getAllLoadedFiles(); - const localHistory = await loadDeleteRenameHistoryTableByVault( - this.db, - this.settings.vaultRandomID - ); - // log.info(local); - // log.info(localHistory); + // getNotice("3/7 Starting to fetch local meta data."); + // this.syncStatus = "getting_local_meta"; + // const local = this.app.vault.getAllLoadedFiles(); + // const localHistory = await loadDeleteRenameHistoryTableByVault( + // this.db, + // this.settings.vaultRandomID + // ); + // // log.info(local); + // // log.info(localHistory); - getNotice("4/7 Checking password correct or not."); - this.syncStatus = "checking_password"; - const passwordCheckResult = await isPasswordOk( - remoteRsp.Contents, - this.settings.password - ); - if (!passwordCheckResult.ok) { - getNotice("something goes wrong while checking password"); - throw Error(passwordCheckResult.reason); - } + // getNotice("4/7 Checking password correct or not."); + // this.syncStatus = "checking_password"; + // const passwordCheckResult = await isPasswordOk( + // remoteRsp.Contents, + // this.settings.password + // ); + // if (!passwordCheckResult.ok) { + // getNotice("something goes wrong while checking password"); + // throw Error(passwordCheckResult.reason); + // } - getNotice("5/7 Starting to generate sync plan."); - this.syncStatus = "generating_plan"; - const syncPlan = await getSyncPlan( - remoteRsp.Contents, - local, - localHistory, - this.db, - this.settings.vaultRandomID, - client.serviceType, - this.settings.password - ); - log.info(syncPlan.mixedStates); // for debugging - await insertSyncPlanRecordByVault( - this.db, - syncPlan, - this.settings.vaultRandomID - ); + // getNotice("5/7 Starting to generate sync plan."); + // this.syncStatus = "generating_plan"; + // const syncPlan = await getSyncPlan( + // remoteRsp.Contents, + // local, + // localHistory, + // this.db, + // this.settings.vaultRandomID, + // client.serviceType, + // this.settings.password + // ); + // log.info(syncPlan.mixedStates); // for debugging + // await insertSyncPlanRecordByVault( + // this.db, + // syncPlan, + // this.settings.vaultRandomID + // ); - // The operations above are read only and kind of safe. - // The operations below begins to write or delete (!!!) something. + // // The operations above are read only and kind of safe. + // // The operations below begins to write or delete (!!!) something. - getNotice("6/7 Remotely Save Sync data exchanging!"); + // getNotice("6/7 Remotely Save Sync data exchanging!"); - this.syncStatus = "syncing"; - await doActualSync( - client, - this.db, - this.settings.vaultRandomID, - this.app.vault, - syncPlan, - this.settings.password, - (i: number, totalCount: number, pathName: string, decision: string) => - self.setCurrSyncMsg(i, totalCount, pathName, decision) - ); + // this.syncStatus = "syncing"; + // await doActualSync( + // client, + // this.db, + // this.settings.vaultRandomID, + // this.app.vault, + // syncPlan, + // this.settings.password, + // (i: number, totalCount: number, pathName: string, decision: string) => + // self.setCurrSyncMsg(i, totalCount, pathName, decision) + // ); getNotice("7/7 Remotely Save finish!"); this.currSyncMsg = ""; @@ -504,6 +504,9 @@ export default class RemotelySavePlugin extends Plugin { if (this.settings.onedrive.authority === "") { this.settings.onedrive.authority = DEFAULT_SETTINGS.onedrive.authority; } + if (this.settings.webdav.manualRecursive === undefined) { + this.settings.webdav.manualRecursive = false; + } } async saveSettings() { diff --git a/src/remoteForWebdav.ts b/src/remoteForWebdav.ts index 90482ff..c6ea009 100644 --- a/src/remoteForWebdav.ts +++ b/src/remoteForWebdav.ts @@ -2,6 +2,9 @@ import { Buffer } from "buffer"; import { Vault } from "obsidian"; import type { FileStat, WebDAVClient } from "webdav/web"; import { AuthType, BufferLike, createClient } from "webdav/web"; +import { Queue } from "@fyears/tsqueue"; +import chunk from "lodash/chunk"; +import flatten from "lodash/flatten"; import type { RemoteItem, WebdavConfig } from "./baseTypes"; import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt"; import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc"; @@ -15,6 +18,7 @@ export const DEFAULT_WEBDAV_CONFIG = { username: "", password: "", authType: "basic", + manualRecursive: false, } as WebdavConfig; const getWebdavPath = (fileOrFolderPath: string, vaultName: string) => { @@ -199,14 +203,51 @@ export const listFromRemote = async ( throw Error("prefix not supported"); } await client.init(); - const contents = (await client.client.getDirectoryContents( - `/${client.vaultName}`, - { - deep: true, - details: false /* no need for verbose details here */, - glob: "/**" /* avoid dot files by using glob */, + + let contents = [] as FileStat[]; + if (client.webdavConfig.manualRecursive) { + // the remote doesn't support infinity propfind, + // we need to do a bfs here + const q = new Queue([`/${client.vaultName}`]); + const CHUNK_SIZE = 10; + while (q.length > 0) { + const itemsToFetch = []; + while (q.length > 0) { + itemsToFetch.push(q.pop()); + } + const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE); + // log.debug(itemsToFetchChunks); + const subContents = [] as FileStat[]; + for (const singleChunk of itemsToFetchChunks) { + const r = singleChunk.map((x) => { + return client.client.getDirectoryContents(x, { + deep: false, + details: false /* no need for verbose details here */, + glob: "/**" /* avoid dot files by using glob */, + }) as Promise; + }); + const r2 = flatten(await Promise.all(r)); + subContents.push(...r2); + } + for (let i = 0; i < subContents.length; ++i) { + const f = subContents[i]; + contents.push(f); + if (f.type === "directory") { + q.push(f.filename); + } + } } - )) as FileStat[]; + } else { + // the remote supports infinity propfind + contents = (await client.client.getDirectoryContents( + `/${client.vaultName}`, + { + deep: true, + details: false /* no need for verbose details here */, + glob: "/**" /* avoid dot files by using glob */, + } + )) as FileStat[]; + } return { Contents: contents.map((x) => fromWebdavItemToRemoteItem(x, client.vaultName) diff --git a/src/settings.ts b/src/settings.ts index d67383a..60fa9ed 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -872,6 +872,32 @@ export class RemotelySaveSettingTab extends PluginSettingTab { }); }); + new Setting(webdavDiv) + .setName("server supports infinity propfind or not") + .setDesc( + "The plugin needs to get all files and folders recursively using probfind. If your webdav server only supports depth='1' (such as NGINX), you need to adjust the setting here, then the plugin consumes more network requests, but better than not working." + ) + .addDropdown((dropdown) => { + dropdown.addOption("infinity", "supports depth='infinity'"); + dropdown.addOption("1", "only supports depth='1'"); + + type Depth = "1" | "infinity"; + dropdown + .setValue( + this.plugin.settings.webdav.manualRecursive === false + ? "infinity" + : "1" + ) + .onChange(async (val: Depth) => { + if (val === "1") { + this.plugin.settings.webdav.manualRecursive = true; + } else if (val === "infinity") { + this.plugin.settings.webdav.manualRecursive = false; + } + await this.plugin.saveSettings(); + }); + }); + new Setting(webdavDiv) .setName("check connectivity") .setDesc("check connectivity")