diff --git a/src/fsWebdav.ts b/src/fsWebdav.ts index 9b8e086..4d68ce3 100644 --- a/src/fsWebdav.ts +++ b/src/fsWebdav.ts @@ -4,6 +4,7 @@ import { getReasonPhrase } from "http-status-codes/build/cjs/utils-functions"; import chunk from "lodash/chunk"; import cloneDeep from "lodash/cloneDeep"; import flatten from "lodash/flatten"; +import { nanoid } from "nanoid"; import { Platform, requestUrl } from "obsidian"; import type { FileStat, @@ -203,6 +204,14 @@ const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string): Entity => { }; }; +const tryEncodeURI = (x: string) => { + if (x.includes("%")) { + // likely encoded before! + return x; + } + return encodeURI(x); +}; + export class FakeFsWebdav extends FakeFs { kind: "webdav"; @@ -212,6 +221,9 @@ export class FakeFsWebdav extends FakeFs { vaultFolderExists: boolean; saveUpdatedConfigFunc: () => Promise; + supportNativePartial: boolean; + isNextcloud: boolean; + constructor( webdavConfig: WebdavConfig, vaultName: string, @@ -220,10 +232,13 @@ export class FakeFsWebdav extends FakeFs { super(); this.kind = "webdav"; this.webdavConfig = cloneDeep(webdavConfig); - this.webdavConfig.address = encodeURI(this.webdavConfig.address); + this.webdavConfig.address = tryEncodeURI(this.webdavConfig.address); this.remoteBaseDir = this.webdavConfig.remoteBaseDir || vaultName || ""; this.vaultFolderExists = false; this.saveUpdatedConfigFunc = saveUpdatedConfigFunc; + + this.supportNativePartial = false; + this.isNextcloud = false; } async _init() { @@ -293,6 +308,43 @@ export class FakeFsWebdav extends FakeFs { ); } } + + await this._checkPartialSupport(); + } + + async _checkPartialSupport() { + const compliance = await this.client.getDAVCompliance( + `/${this.remoteBaseDir}/` + ); + + for (const c of compliance.compliance) { + if ( + c.toLocaleLowerCase().includes("nextcloud") && + this.webdavConfig.username !== "" && + this.webdavConfig.password !== "" + ) { + // nextcloud AND with an account + this.isNextcloud = true; + console.debug(`isNextcloud=true`); + return true; + } + } + + // taken from https://github.com/perry-mitchell/webdav-client/blob/master/source/operations/partialUpdateFileContents.ts + // which is under MIT license + if ( + (compliance.server.includes("Apache") && + compliance.compliance.includes( + "" + )) || + compliance.compliance.includes("sabredav-partialupdate") + ) { + this.supportNativePartial = true; + console.debug(`supportNativePartial=true`); + return true; + } + + return false; } async walk(): Promise { @@ -422,6 +474,40 @@ export class FakeFsWebdav extends FakeFs { content: ArrayBuffer, mtime: number, ctime: number + ): Promise { + // less than 10 MB + if (content.byteLength <= 10 * 1024 * 1024) { + return await this._writeFileFromRootFull(key, content, mtime, ctime); + } + + // larger than 10 MB, try to upload by chunks + try { + if (this.isNextcloud) { + return await this._writeFileFromRootNextcloud( + key, + content, + mtime, + ctime + ); + } else if (this.supportNativePartial) { + return await this._writeFileFromRootNativePartial( + key, + content, + mtime, + ctime + ); + } + throw Error(`no partial upload / update`); + } catch (e) { + return await this._writeFileFromRootFull(key, content, mtime, ctime); + } + } + + async _writeFileFromRootFull( + key: string, + content: ArrayBuffer, + mtime: number, + ctime: number ): Promise { await this.client.putFileContents(key, content, { overwrite: true, @@ -432,6 +518,136 @@ export class FakeFsWebdav extends FakeFs { return await this._statFromRoot(key); } + /** + * https://docs.nextcloud.com/server/latest/developer_manual/client_apis/WebDAV/chunking.html + * @param key + * @param content + * @param mtime + * @param ctime + * @returns + */ + async _writeFileFromRootNextcloud( + key: string, + content: ArrayBuffer, + mtime: number, + ctime: number + ): Promise { + if (key.endsWith("/")) { + throw Error( + `key=${key} should not have tailing slash in _writeFileFromRootNextcloud` + ); + } + const destUrl = `${this.webdavConfig.address}/${encodeURI(key)}`; + console.debug(`destUrl=${destUrl}`); + const tmpFolder = `_${key}-${nanoid()}`; + console.debug(`tmpFolder=${tmpFolder}`); + const tmpFolderUrl = `${this.webdavConfig.address}/${encodeURI(tmpFolder)}`; + console.debug(`tmpFolderUrl=${tmpFolderUrl}`); + + // create folder + await this.client.createDirectory(tmpFolder, { + headers: { + Destination: destUrl, + }, + }); + console.debug(`finish creating folder`); + + // upload by chunks + const size_6mb = 6 * 1024 * 1024; + let tmpFileIdx = 1; // a number between 1 and 10000 + let startInclusive = 0; + let endInclusive = Math.min(size_6mb, content.byteLength); + do { + const tmpFileName = `${tmpFileIdx}`.padStart(5, "0"); + const tmpFileNameWithFolder = `${tmpFolder}/${tmpFileName}`; + console.debug( + `start to upload chunk ${tmpFileIdx} to ${tmpFileNameWithFolder} with startInclusive=${startInclusive}, endInclusive=${endInclusive}` + ); + await this.client.putFileContents( + tmpFileNameWithFolder, + content.slice(startInclusive, endInclusive - 1), + { + headers: { + Destination: destUrl, + "OC-Total-Length": `${content.byteLength}`, + }, + } + ); + tmpFileIdx += 1; + startInclusive = Math.min(startInclusive + size_6mb, content.byteLength); + endInclusive = Math.min(endInclusive + size_6mb, content.byteLength); + } while (startInclusive < content.byteLength); + console.debug(`finish upload all chunks`); + + // move to assemble + try { + const fakeFileToMoveUrl = `${tmpFolderUrl}/.file`; + console.debug(`fakeFileToMoveUrl=${fakeFileToMoveUrl}`); + await this.client.customRequest(fakeFileToMoveUrl, { + method: "MOVE", + headers: { + Destination: destUrl, + "OC-Total-Length": `${content.byteLength}`, + }, + }); + console.debug(`finish moving file`); + } catch (e) { + // sometimes the server returns 404 but actually it works, + // we ignore the error. + console.error( + `while assembling chunks of nextcloud, some errors occur but we ignore them:` + ); + console.error(e); + } + // TODO: setting X-OC-Mtime + + // stat + const k = await this._statFromRoot(key); + if (k.sizeRaw !== content.byteLength) { + // we failed! + this.isNextcloud = false; // give up next time! + throw Error(`unable to upload file ${key} by chunks to nextcloud`); + } + + // clean up! + try { + await this.client.deleteFile(tmpFolder); + } catch (e) { + // the folde might exist or not, still ignore the error + } + + return k; + } + + async _writeFileFromRootNativePartial( + key: string, + content: ArrayBuffer, + mtime: number, + ctime: number + ): Promise { + // firstly upload a 0-byte data + await this._writeFileFromRootFull(key, new ArrayBuffer(0), mtime, ctime); + + // then "update" by chunks + const size_6mb = 6 * 1024 * 1024; + let startInclusive = 0; + let endInclusive = Math.min(size_6mb, content.byteLength); + do { + await this.client.partialUpdateFileContents( + key, + startInclusive, + endInclusive, + content.slice(startInclusive, endInclusive - 1) + ); + + startInclusive = Math.min(startInclusive + size_6mb, content.byteLength); + endInclusive = Math.min(endInclusive + size_6mb, content.byteLength); + } while (startInclusive < content.byteLength); + + // lastly return + return await this._statFromRoot(key); + } + async readFile(key: string): Promise { if (key.endsWith("/")) { throw Error(`you should not call readFile on ${key}`); @@ -484,6 +700,8 @@ export class FakeFsWebdav extends FakeFs { try { await this._init(); + const c = await this.client.getDAVCompliance(`/${this.remoteBaseDir}/`); + console.debug(c); const results = await this._statFromRoot(`/${this.remoteBaseDir}/`); if (results === undefined) { const err = "results is undefined";