a large semi-rewrite of fs logic
This commit is contained in:
parent
5ce350ba41
commit
f33fa26c03
|
@ -36,6 +36,7 @@ esbuild
|
||||||
"net",
|
"net",
|
||||||
"http",
|
"http",
|
||||||
"https",
|
"https",
|
||||||
|
"vm",
|
||||||
// ...builtins
|
// ...builtins
|
||||||
],
|
],
|
||||||
inject: ["./esbuild.injecthelper.mjs"],
|
inject: ["./esbuild.injecthelper.mjs"],
|
||||||
|
|
|
@ -16,7 +16,9 @@
|
||||||
"process": "process/browser",
|
"process": "process/browser",
|
||||||
"stream": "stream-browserify",
|
"stream": "stream-browserify",
|
||||||
"crypto": "crypto-browserify",
|
"crypto": "crypto-browserify",
|
||||||
"url": "url/"
|
"url": "url/",
|
||||||
|
"fs": false,
|
||||||
|
"vm": false
|
||||||
},
|
},
|
||||||
"source": "main.ts",
|
"source": "main.ts",
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
|
@ -59,7 +61,7 @@
|
||||||
"@aws-sdk/lib-storage": "^3.474.0",
|
"@aws-sdk/lib-storage": "^3.474.0",
|
||||||
"@aws-sdk/signature-v4-crt": "^3.474.0",
|
"@aws-sdk/signature-v4-crt": "^3.474.0",
|
||||||
"@aws-sdk/types": "^3.468.0",
|
"@aws-sdk/types": "^3.468.0",
|
||||||
"@azure/msal-node": "^2.6.0",
|
"@azure/msal-node": "^2.7.0",
|
||||||
"@fyears/rclone-crypt": "^0.0.7",
|
"@fyears/rclone-crypt": "^0.0.7",
|
||||||
"@fyears/tsqueue": "^1.0.1",
|
"@fyears/tsqueue": "^1.0.1",
|
||||||
"@microsoft/microsoft-graph-client": "^3.0.7",
|
"@microsoft/microsoft-graph-client": "^3.0.7",
|
||||||
|
|
|
@ -1,215 +0,0 @@
|
||||||
import { CipherMethodType } from "./baseTypes";
|
|
||||||
import * as openssl from "./encryptOpenSSL";
|
|
||||||
import * as rclone from "./encryptRClone";
|
|
||||||
import { isVaildText } from "./misc";
|
|
||||||
|
|
||||||
export class Cipher {
|
|
||||||
readonly password: string;
|
|
||||||
readonly method: CipherMethodType;
|
|
||||||
cipherRClone?: rclone.CipherRclone;
|
|
||||||
constructor(password: string, method: CipherMethodType) {
|
|
||||||
this.password = password ?? "";
|
|
||||||
this.method = method;
|
|
||||||
|
|
||||||
if (method === "rclone-base64") {
|
|
||||||
this.cipherRClone = new rclone.CipherRclone(password, 5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
closeResources() {
|
|
||||||
if (this.method === "rclone-base64" && this.cipherRClone !== undefined) {
|
|
||||||
this.cipherRClone.closeResources();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
isPasswordEmpty() {
|
|
||||||
return this.password === "";
|
|
||||||
}
|
|
||||||
|
|
||||||
isFolderAware() {
|
|
||||||
if (this.method === "openssl-base64") {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (this.method === "rclone-base64") {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
throw Error(`no idea about isFolderAware for method=${this.method}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
async encryptContent(content: ArrayBuffer) {
|
|
||||||
// console.debug("start encryptContent");
|
|
||||||
if (this.password === "") {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
if (this.method === "openssl-base64") {
|
|
||||||
const res = await openssl.encryptArrayBuffer(content, this.password);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot encrypt content`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else if (this.method === "rclone-base64") {
|
|
||||||
const res =
|
|
||||||
await this.cipherRClone!.encryptContentByCallingWorker(content);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot encrypt content`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported encrypt method=${this.method}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async decryptContent(content: ArrayBuffer) {
|
|
||||||
// console.debug("start decryptContent");
|
|
||||||
if (this.password === "") {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
if (this.method === "openssl-base64") {
|
|
||||||
const res = await openssl.decryptArrayBuffer(content, this.password);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot decrypt content`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else if (this.method === "rclone-base64") {
|
|
||||||
const res =
|
|
||||||
await this.cipherRClone!.decryptContentByCallingWorker(content);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot decrypt content`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported decrypt method=${this.method}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async encryptName(name: string) {
|
|
||||||
// console.debug("start encryptName");
|
|
||||||
if (this.password === "") {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
if (this.method === "openssl-base64") {
|
|
||||||
const res = await openssl.encryptStringToBase64url(name, this.password);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot encrypt name=${name}`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else if (this.method === "rclone-base64") {
|
|
||||||
const res = await this.cipherRClone!.encryptNameByCallingWorker(name);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot encrypt name=${name}`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported encrypt method=${this.method}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async decryptName(name: string): Promise<string> {
|
|
||||||
// console.debug("start decryptName");
|
|
||||||
if (this.password === "") {
|
|
||||||
return name;
|
|
||||||
}
|
|
||||||
if (this.method === "openssl-base64") {
|
|
||||||
if (name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
|
||||||
// backward compitable with the openssl-base32
|
|
||||||
try {
|
|
||||||
const res = await openssl.decryptBase32ToString(name, this.password);
|
|
||||||
if (res !== undefined && isVaildText(res)) {
|
|
||||||
return res;
|
|
||||||
} else {
|
|
||||||
throw Error(`cannot decrypt name=${name}`);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
throw Error(`cannot decrypt name=${name}`);
|
|
||||||
}
|
|
||||||
} else if (name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
|
|
||||||
try {
|
|
||||||
const res = await openssl.decryptBase64urlToString(
|
|
||||||
name,
|
|
||||||
this.password
|
|
||||||
);
|
|
||||||
if (res !== undefined && isVaildText(res)) {
|
|
||||||
return res;
|
|
||||||
} else {
|
|
||||||
throw Error(`cannot decrypt name=${name}`);
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
throw Error(`cannot decrypt name=${name}`);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw Error(
|
|
||||||
`method=${this.method} but the name=${name}, likely mismatch`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else if (this.method === "rclone-base64") {
|
|
||||||
const res = await this.cipherRClone!.decryptNameByCallingWorker(name);
|
|
||||||
if (res === undefined) {
|
|
||||||
throw Error(`cannot decrypt name=${name}`);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported decrypt method=${this.method}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
getSizeFromOrigToEnc(x: number) {
|
|
||||||
if (this.password === "") {
|
|
||||||
return x;
|
|
||||||
}
|
|
||||||
if (this.method === "openssl-base64") {
|
|
||||||
return openssl.getSizeFromOrigToEnc(x);
|
|
||||||
} else if (this.method === "rclone-base64") {
|
|
||||||
return rclone.getSizeFromOrigToEnc(x);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported encrypt method=${this.method}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* quick guess, no actual decryption here
|
|
||||||
* @param name
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
static isLikelyOpenSSLEncryptedName(name: string): boolean {
|
|
||||||
if (
|
|
||||||
name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE32) ||
|
|
||||||
name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE64URL)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* quick guess, no actual decryption here
|
|
||||||
* @param name
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
static isLikelyEncryptedName(name: string): boolean {
|
|
||||||
return Cipher.isLikelyOpenSSLEncryptedName(name);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* quick guess, no actual decryption here, only openssl can be guessed here
|
|
||||||
* @param name
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
static isLikelyEncryptedNameNotMatchMethod(
|
|
||||||
name: string,
|
|
||||||
method: CipherMethodType
|
|
||||||
): boolean {
|
|
||||||
if (
|
|
||||||
Cipher.isLikelyOpenSSLEncryptedName(name) &&
|
|
||||||
method !== "openssl-base64"
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
!Cipher.isLikelyOpenSSLEncryptedName(name) &&
|
|
||||||
method === "openssl-base64"
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
import { Entity } from "./baseTypes";
|
||||||
|
|
||||||
|
export abstract class FakeFs {
|
||||||
|
abstract kind: string;
|
||||||
|
abstract walk(): Promise<Entity[]>;
|
||||||
|
abstract stat(key: string): Promise<Entity>;
|
||||||
|
abstract mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity>;
|
||||||
|
abstract writeFile(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity>;
|
||||||
|
abstract readFile(key: string): Promise<ArrayBuffer>;
|
||||||
|
abstract rm(key: string): Promise<void>;
|
||||||
|
abstract checkConnect(callbackFunc?: any): Promise<boolean>;
|
||||||
|
abstract getUserDisplayName(): Promise<string>;
|
||||||
|
abstract revokeAuth(): Promise<any>;
|
||||||
|
}
|
|
@ -1,25 +1,21 @@
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
import { Dropbox, DropboxAuth } from "dropbox";
|
import { Dropbox, DropboxAuth } from "dropbox";
|
||||||
import type { files, DropboxResponseError, DropboxResponse } from "dropbox";
|
import type { files, DropboxResponseError, DropboxResponse } from "dropbox";
|
||||||
import { Vault } from "obsidian";
|
|
||||||
import * as path from "path";
|
|
||||||
import {
|
import {
|
||||||
DropboxConfig,
|
DropboxConfig,
|
||||||
Entity,
|
|
||||||
COMMAND_CALLBACK_DROPBOX,
|
COMMAND_CALLBACK_DROPBOX,
|
||||||
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
|
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
|
||||||
UploadedType,
|
Entity,
|
||||||
} from "./baseTypes";
|
} from "./baseTypes";
|
||||||
|
import random from "lodash/random";
|
||||||
import {
|
import {
|
||||||
bufferToArrayBuffer,
|
bufferToArrayBuffer,
|
||||||
delay,
|
delay,
|
||||||
fixEntityListCasesInplace,
|
|
||||||
getFolderLevels,
|
getFolderLevels,
|
||||||
|
getParentFolder,
|
||||||
hasEmojiInText,
|
hasEmojiInText,
|
||||||
headersToRecord,
|
headersToRecord,
|
||||||
mkdirpInVault,
|
|
||||||
} from "./misc";
|
} from "./misc";
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
import { random } from "lodash";
|
|
||||||
|
|
||||||
export { Dropbox } from "dropbox";
|
export { Dropbox } from "dropbox";
|
||||||
|
|
||||||
|
@ -34,10 +30,7 @@ export const DEFAULT_DROPBOX_CONFIG: DropboxConfig = {
|
||||||
credentialsShouldBeDeletedAtTime: 0,
|
credentialsShouldBeDeletedAtTime: 0,
|
||||||
};
|
};
|
||||||
|
|
||||||
export const getDropboxPath = (
|
const getDropboxPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
||||||
fileOrFolderPath: string,
|
|
||||||
remoteBaseDir: string
|
|
||||||
) => {
|
|
||||||
let key = fileOrFolderPath;
|
let key = fileOrFolderPath;
|
||||||
if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
|
if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
|
||||||
// special
|
// special
|
||||||
|
@ -84,20 +77,22 @@ const fromDropboxItemToEntity = (
|
||||||
|
|
||||||
if (x[".tag"] === "folder") {
|
if (x[".tag"] === "folder") {
|
||||||
return {
|
return {
|
||||||
|
key: key,
|
||||||
keyRaw: key,
|
keyRaw: key,
|
||||||
|
size: 0,
|
||||||
sizeRaw: 0,
|
sizeRaw: 0,
|
||||||
etag: `${x.id}\t`,
|
|
||||||
} as Entity;
|
} as Entity;
|
||||||
} else if (x[".tag"] === "file") {
|
} else if (x[".tag"] === "file") {
|
||||||
const mtimeCli = Date.parse(x.client_modified).valueOf();
|
const mtimeCli = Date.parse(x.client_modified).valueOf();
|
||||||
const mtimeSvr = Date.parse(x.server_modified).valueOf();
|
const mtimeSvr = Date.parse(x.server_modified).valueOf();
|
||||||
return {
|
return {
|
||||||
|
key: key,
|
||||||
keyRaw: key,
|
keyRaw: key,
|
||||||
mtimeCli: mtimeCli,
|
mtimeCli: mtimeCli,
|
||||||
mtimeSvr: mtimeSvr,
|
mtimeSvr: mtimeSvr,
|
||||||
|
size: x.size,
|
||||||
sizeRaw: x.size,
|
sizeRaw: x.size,
|
||||||
hash: x.content_hash,
|
hash: x.content_hash,
|
||||||
etag: `${x.id}\t${x.content_hash}`,
|
|
||||||
} as Entity;
|
} as Entity;
|
||||||
} else {
|
} else {
|
||||||
// x[".tag"] === "deleted"
|
// x[".tag"] === "deleted"
|
||||||
|
@ -105,6 +100,132 @@ const fromDropboxItemToEntity = (
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* https://github.com/remotely-save/remotely-save/issues/567
|
||||||
|
* https://www.dropboxforum.com/t5/Dropbox-API-Support-Feedback/Case-Sensitivity-in-API-2/td-p/191279
|
||||||
|
* @param entities
|
||||||
|
*/
|
||||||
|
export const fixEntityListCasesInplace = (entities: { key?: string }[]) => {
|
||||||
|
for (const iterator of entities) {
|
||||||
|
if (iterator.key === undefined) {
|
||||||
|
throw Error(`dropbox list should all have key, but meet undefined`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
entities.sort((a, b) => a.key!.length - b.key!.length);
|
||||||
|
// console.log(JSON.stringify(entities,null,2));
|
||||||
|
|
||||||
|
const caseMapping: Record<string, string> = { "": "" };
|
||||||
|
for (const e of entities) {
|
||||||
|
// console.log(`looking for: ${JSON.stringify(e, null, 2)}`);
|
||||||
|
|
||||||
|
let parentFolder = getParentFolder(e.key!);
|
||||||
|
if (parentFolder === "/") {
|
||||||
|
parentFolder = "";
|
||||||
|
}
|
||||||
|
const parentFolderLower = parentFolder.toLocaleLowerCase();
|
||||||
|
const segs = e.key!.split("/");
|
||||||
|
if (e.key!.endsWith("/")) {
|
||||||
|
// folder
|
||||||
|
if (caseMapping.hasOwnProperty(parentFolderLower)) {
|
||||||
|
const newKey = `${caseMapping[parentFolderLower]}${segs
|
||||||
|
.slice(-2)
|
||||||
|
.join("/")}`;
|
||||||
|
caseMapping[newKey.toLocaleLowerCase()] = newKey;
|
||||||
|
e.key = newKey;
|
||||||
|
// console.log(JSON.stringify(caseMapping,null,2));
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
throw Error(`${parentFolder} doesn't have cases record??`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// file
|
||||||
|
if (caseMapping.hasOwnProperty(parentFolderLower)) {
|
||||||
|
const newKey = `${caseMapping[parentFolderLower]}${segs
|
||||||
|
.slice(-1)
|
||||||
|
.join("/")}`;
|
||||||
|
e.key = newKey;
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
throw Error(`${parentFolder} doesn't have cases record??`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return entities;
|
||||||
|
};
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// Other usual common methods
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
interface ErrSubType {
|
||||||
|
error: {
|
||||||
|
retry_after: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function retryReq<T>(
|
||||||
|
reqFunc: () => Promise<DropboxResponse<T>>,
|
||||||
|
extraHint: string = ""
|
||||||
|
): Promise<DropboxResponse<T> | undefined> {
|
||||||
|
const waitSeconds = [1, 2, 4, 8]; // hard code exponential backoff
|
||||||
|
for (let idx = 0; idx < waitSeconds.length; ++idx) {
|
||||||
|
try {
|
||||||
|
if (idx !== 0) {
|
||||||
|
console.warn(
|
||||||
|
`${extraHint === "" ? "" : extraHint + ": "}The ${
|
||||||
|
idx + 1
|
||||||
|
}-th try starts at time ${Date.now()}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return await reqFunc();
|
||||||
|
} catch (e: unknown) {
|
||||||
|
const err = e as DropboxResponseError<ErrSubType>;
|
||||||
|
if (err.status === undefined) {
|
||||||
|
// then the err is not DropboxResponseError
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (err.status !== 429) {
|
||||||
|
// then the err is not "too many requests", give up
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (idx === waitSeconds.length - 1) {
|
||||||
|
// the last retry also failed, give up
|
||||||
|
throw new Error(
|
||||||
|
`${
|
||||||
|
extraHint === "" ? "" : extraHint + ": "
|
||||||
|
}"429 too many requests", after retrying for ${
|
||||||
|
idx + 1
|
||||||
|
} times still failed.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers = headersToRecord(err.headers);
|
||||||
|
const svrSec =
|
||||||
|
err.error.error.retry_after ||
|
||||||
|
parseInt(headers["retry-after"] || "1") ||
|
||||||
|
1;
|
||||||
|
const fallbackSec = waitSeconds[idx];
|
||||||
|
const secMin = Math.max(svrSec, fallbackSec);
|
||||||
|
const secMax = Math.max(secMin * 1.8, 2);
|
||||||
|
console.warn(
|
||||||
|
`${
|
||||||
|
extraHint === "" ? "" : extraHint + ": "
|
||||||
|
}We have "429 too many requests" error of ${
|
||||||
|
idx + 1
|
||||||
|
}-th try, at time ${Date.now()}, and wait for ${secMin} ~ ${secMax} seconds to retry. Original info: ${JSON.stringify(
|
||||||
|
err.error,
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)}`
|
||||||
|
);
|
||||||
|
await delay(random(secMin * 1000, secMax * 1000));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
// Dropbox authorization using PKCE
|
// Dropbox authorization using PKCE
|
||||||
// see https://dropbox.tech/developers/pkce--what-and-why-
|
// see https://dropbox.tech/developers/pkce--what-and-why-
|
||||||
|
@ -228,94 +349,33 @@ export const setConfigBySuccessfullAuthInplace = async (
|
||||||
};
|
};
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
// Other usual common methods
|
// real exported interface
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
interface ErrSubType {
|
export class FakeFsDropbox extends FakeFs {
|
||||||
error: {
|
kind: "dropbox";
|
||||||
retry_after: number;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function retryReq<T>(
|
|
||||||
reqFunc: () => Promise<DropboxResponse<T>>,
|
|
||||||
extraHint: string = ""
|
|
||||||
): Promise<DropboxResponse<T> | undefined> {
|
|
||||||
const waitSeconds = [1, 2, 4, 8]; // hard code exponential backoff
|
|
||||||
for (let idx = 0; idx < waitSeconds.length; ++idx) {
|
|
||||||
try {
|
|
||||||
if (idx !== 0) {
|
|
||||||
console.warn(
|
|
||||||
`${extraHint === "" ? "" : extraHint + ": "}The ${
|
|
||||||
idx + 1
|
|
||||||
}-th try starts at time ${Date.now()}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return await reqFunc();
|
|
||||||
} catch (e: unknown) {
|
|
||||||
const err = e as DropboxResponseError<ErrSubType>;
|
|
||||||
if (err.status === undefined) {
|
|
||||||
// then the err is not DropboxResponseError
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
if (err.status !== 429) {
|
|
||||||
// then the err is not "too many requests", give up
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (idx === waitSeconds.length - 1) {
|
|
||||||
// the last retry also failed, give up
|
|
||||||
throw new Error(
|
|
||||||
`${
|
|
||||||
extraHint === "" ? "" : extraHint + ": "
|
|
||||||
}"429 too many requests", after retrying for ${
|
|
||||||
idx + 1
|
|
||||||
} times still failed.`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const headers = headersToRecord(err.headers);
|
|
||||||
const svrSec =
|
|
||||||
err.error.error.retry_after ||
|
|
||||||
parseInt(headers["retry-after"] || "1") ||
|
|
||||||
1;
|
|
||||||
const fallbackSec = waitSeconds[idx];
|
|
||||||
const secMin = Math.max(svrSec, fallbackSec);
|
|
||||||
const secMax = Math.max(secMin * 1.8, 2);
|
|
||||||
console.warn(
|
|
||||||
`${
|
|
||||||
extraHint === "" ? "" : extraHint + ": "
|
|
||||||
}We have "429 too many requests" error of ${
|
|
||||||
idx + 1
|
|
||||||
}-th try, at time ${Date.now()}, and wait for ${secMin} ~ ${secMax} seconds to retry. Original info: ${JSON.stringify(
|
|
||||||
err.error,
|
|
||||||
null,
|
|
||||||
2
|
|
||||||
)}`
|
|
||||||
);
|
|
||||||
await delay(random(secMin * 1000, secMax * 1000));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export class WrappedDropboxClient {
|
|
||||||
dropboxConfig: DropboxConfig;
|
dropboxConfig: DropboxConfig;
|
||||||
remoteBaseDir: string;
|
remoteBaseDir: string;
|
||||||
saveUpdatedConfigFunc: () => Promise<any>;
|
saveUpdatedConfigFunc: () => Promise<any>;
|
||||||
dropbox!: Dropbox;
|
dropbox!: Dropbox;
|
||||||
vaultFolderExists: boolean;
|
vaultFolderExists: boolean;
|
||||||
|
foldersCreatedBefore: Set<string>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
dropboxConfig: DropboxConfig,
|
dropboxConfig: DropboxConfig,
|
||||||
remoteBaseDir: string,
|
vaultName: string,
|
||||||
saveUpdatedConfigFunc: () => Promise<any>
|
saveUpdatedConfigFunc: () => Promise<any>
|
||||||
) {
|
) {
|
||||||
|
super();
|
||||||
|
this.kind = "dropbox";
|
||||||
this.dropboxConfig = dropboxConfig;
|
this.dropboxConfig = dropboxConfig;
|
||||||
this.remoteBaseDir = remoteBaseDir;
|
this.remoteBaseDir = this.dropboxConfig.remoteBaseDir || vaultName || "";
|
||||||
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
||||||
this.vaultFolderExists = false;
|
this.vaultFolderExists = false;
|
||||||
|
this.foldersCreatedBefore = new Set();
|
||||||
}
|
}
|
||||||
|
|
||||||
init = async () => {
|
async _init() {
|
||||||
// check token
|
// check token
|
||||||
if (
|
if (
|
||||||
this.dropboxConfig.accessToken === "" ||
|
this.dropboxConfig.accessToken === "" ||
|
||||||
|
@ -388,389 +448,292 @@ export class WrappedDropboxClient {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.dropbox;
|
return this;
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param dropboxConfig
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
export const getDropboxClient = (
|
|
||||||
dropboxConfig: DropboxConfig,
|
|
||||||
remoteBaseDir: string,
|
|
||||||
saveUpdatedConfigFunc: () => Promise<any>
|
|
||||||
) => {
|
|
||||||
return new WrappedDropboxClient(
|
|
||||||
dropboxConfig,
|
|
||||||
remoteBaseDir,
|
|
||||||
saveUpdatedConfigFunc
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getRemoteMeta = async (
|
|
||||||
client: WrappedDropboxClient,
|
|
||||||
remotePath: string
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
// if (remotePath === "" || remotePath === "/") {
|
|
||||||
// // filesGetMetadata doesn't support root folder
|
|
||||||
// // we instead try to list files
|
|
||||||
// // if no error occurs, we ensemble a fake result.
|
|
||||||
// const rsp = await retryReq(() =>
|
|
||||||
// client.dropbox.filesListFolder({
|
|
||||||
// path: `/${client.remoteBaseDir}`,
|
|
||||||
// recursive: false, // don't need to recursive here
|
|
||||||
// })
|
|
||||||
// );
|
|
||||||
// if (rsp.status !== 200) {
|
|
||||||
// throw Error(JSON.stringify(rsp));
|
|
||||||
// }
|
|
||||||
// return {
|
|
||||||
// key: remotePath,
|
|
||||||
// lastModified: undefined,
|
|
||||||
// size: 0,
|
|
||||||
// remoteType: "dropbox",
|
|
||||||
// etag: undefined,
|
|
||||||
// } as Entity;
|
|
||||||
// }
|
|
||||||
|
|
||||||
const rsp = await retryReq(() =>
|
|
||||||
client.dropbox.filesGetMetadata({
|
|
||||||
path: remotePath,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
if (rsp === undefined) {
|
|
||||||
throw Error("dropbox.filesGetMetadata undefinded");
|
|
||||||
}
|
}
|
||||||
if (rsp.status !== 200) {
|
|
||||||
throw Error(JSON.stringify(rsp));
|
async walk(): Promise<Entity[]> {
|
||||||
|
await this._init();
|
||||||
|
|
||||||
|
let res = await this.dropbox.filesListFolder({
|
||||||
|
path: `/${this.remoteBaseDir}`,
|
||||||
|
recursive: true,
|
||||||
|
include_deleted: false,
|
||||||
|
limit: 1000,
|
||||||
|
});
|
||||||
|
if (res.status !== 200) {
|
||||||
|
throw Error(JSON.stringify(res));
|
||||||
|
}
|
||||||
|
// console.info(res);
|
||||||
|
|
||||||
|
const contents = res.result.entries;
|
||||||
|
const unifiedContents = contents
|
||||||
|
.filter((x) => x[".tag"] !== "deleted")
|
||||||
|
.filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
|
||||||
|
.map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
|
||||||
|
|
||||||
|
while (res.result.has_more) {
|
||||||
|
res = await this.dropbox.filesListFolderContinue({
|
||||||
|
cursor: res.result.cursor,
|
||||||
|
});
|
||||||
|
if (res.status !== 200) {
|
||||||
|
throw Error(JSON.stringify(res));
|
||||||
|
}
|
||||||
|
|
||||||
|
const contents2 = res.result.entries;
|
||||||
|
const unifiedContents2 = contents2
|
||||||
|
.filter((x) => x[".tag"] !== "deleted")
|
||||||
|
.filter((x) => x.path_display !== `/${this.remoteBaseDir}`)
|
||||||
|
.map((x) => fromDropboxItemToEntity(x, this.remoteBaseDir));
|
||||||
|
unifiedContents.push(...unifiedContents2);
|
||||||
|
}
|
||||||
|
|
||||||
|
fixEntityListCasesInplace(unifiedContents);
|
||||||
|
|
||||||
|
return unifiedContents;
|
||||||
}
|
}
|
||||||
return fromDropboxItemToEntity(rsp.result, client.remoteBaseDir);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const uploadToRemote = async (
|
async stat(key: string): Promise<Entity> {
|
||||||
client: WrappedDropboxClient,
|
await this._init();
|
||||||
fileOrFolderPath: string,
|
return await this._statFromRoot(getDropboxPath(key, this.remoteBaseDir));
|
||||||
vault: Vault | undefined,
|
}
|
||||||
isRecursively: boolean,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
foldersCreatedBefore: Set<string> | undefined = undefined,
|
|
||||||
uploadRaw: boolean = false,
|
|
||||||
rawContent: string | ArrayBuffer = "",
|
|
||||||
rawContentMTime: number = 0,
|
|
||||||
rawContentCTime: number = 0
|
|
||||||
): Promise<UploadedType> => {
|
|
||||||
await client.init();
|
|
||||||
|
|
||||||
let uploadFile = fileOrFolderPath;
|
async _statFromRoot(key: string): Promise<Entity> {
|
||||||
if (!cipher.isPasswordEmpty()) {
|
// if (key === "" || key === "/") {
|
||||||
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
// // filesGetMetadata doesn't support root folder
|
||||||
throw Error(
|
// // we instead try to list files
|
||||||
`uploadToRemote(dropbox) you have password but remoteEncryptedKey is empty!`
|
// // if no error occurs, we ensemble a fake result.
|
||||||
|
// const rsp = await retryReq(() =>
|
||||||
|
// client.dropbox.filesListFolder({
|
||||||
|
// path: `/${client.key}`,
|
||||||
|
// recursive: false, // don't need to recursive here
|
||||||
|
// })
|
||||||
|
// );
|
||||||
|
// if (rsp.status !== 200) {
|
||||||
|
// throw Error(JSON.stringify(rsp));
|
||||||
|
// }
|
||||||
|
// return {
|
||||||
|
// key: remotePath,
|
||||||
|
// lastModified: undefined,
|
||||||
|
// size: 0,
|
||||||
|
// remoteType: "dropbox",
|
||||||
|
// etag: undefined,
|
||||||
|
// } as Entity;
|
||||||
|
// }
|
||||||
|
|
||||||
|
const rsp = await retryReq(() =>
|
||||||
|
this.dropbox.filesGetMetadata({
|
||||||
|
path: key,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
if (rsp === undefined) {
|
||||||
|
throw Error("dropbox.filesGetMetadata undefinded");
|
||||||
|
}
|
||||||
|
if (rsp.status !== 200) {
|
||||||
|
throw Error(JSON.stringify(rsp));
|
||||||
|
}
|
||||||
|
return fromDropboxItemToEntity(rsp.result, this.remoteBaseDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
async mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity> {
|
||||||
|
if (!key.endsWith("/")) {
|
||||||
|
throw Error(`you should not call mkdir on ${key}`);
|
||||||
|
}
|
||||||
|
await this._init();
|
||||||
|
|
||||||
|
const uploadFile = getDropboxPath(key, this.remoteBaseDir);
|
||||||
|
|
||||||
|
return await this._mkdirFromRoot(uploadFile, mtime, ctime);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _mkdirFromRoot(
|
||||||
|
key: string,
|
||||||
|
mtime?: number,
|
||||||
|
ctime?: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (hasEmojiInText(key)) {
|
||||||
|
throw new Error(
|
||||||
|
`${key}: Error: Dropbox does not support emoji in file / folder names.`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
uploadFile = remoteEncryptedKey;
|
if (this.foldersCreatedBefore?.has(key)) {
|
||||||
|
// created, pass
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
await retryReq(
|
||||||
|
() =>
|
||||||
|
this.dropbox.filesCreateFolderV2({
|
||||||
|
path: key,
|
||||||
|
}),
|
||||||
|
key // just a hint
|
||||||
|
);
|
||||||
|
this.foldersCreatedBefore?.add(key);
|
||||||
|
} catch (e: unknown) {
|
||||||
|
const err = e as DropboxResponseError<files.CreateFolderError>;
|
||||||
|
if (err.status === undefined) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (err.status === 409) {
|
||||||
|
// pass
|
||||||
|
this.foldersCreatedBefore?.add(key);
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return await this._statFromRoot(key);
|
||||||
}
|
}
|
||||||
uploadFile = getDropboxPath(uploadFile, client.remoteBaseDir);
|
|
||||||
|
|
||||||
if (hasEmojiInText(uploadFile)) {
|
async writeFile(
|
||||||
throw new Error(
|
key: string,
|
||||||
`${uploadFile}: Error: Dropbox does not support emoji in file / folder names.`
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
throw Error(`you should not call writeFile on ${key}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
await this._init();
|
||||||
|
const uploadFile = getDropboxPath(key, this.remoteBaseDir);
|
||||||
|
|
||||||
|
return await this._writeFileFromRoot(
|
||||||
|
uploadFile,
|
||||||
|
content,
|
||||||
|
mtime,
|
||||||
|
ctime,
|
||||||
|
key
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mtime = 0;
|
async _writeFileFromRoot(
|
||||||
let ctime = 0;
|
key: string,
|
||||||
const s = await vault?.adapter?.stat(fileOrFolderPath);
|
content: ArrayBuffer,
|
||||||
if (s !== undefined && s !== null) {
|
mtime: number,
|
||||||
mtime = Math.floor(s.mtime / 1000.0) * 1000;
|
ctime: number,
|
||||||
ctime = Math.floor(s.ctime / 1000.0) * 1000;
|
origKey: string
|
||||||
}
|
): Promise<Entity> {
|
||||||
const mtimeStr = new Date(mtime).toISOString().replace(/\.\d{3}Z$/, "Z");
|
if (hasEmojiInText(origKey)) {
|
||||||
|
throw new Error(
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
`${origKey}: Error: Dropbox does not support emoji in file / folder names.`
|
||||||
|
|
||||||
if (isFolder && isRecursively) {
|
|
||||||
throw Error("upload function doesn't implement recursive function yet!");
|
|
||||||
} else if (isFolder && !isRecursively) {
|
|
||||||
if (uploadRaw) {
|
|
||||||
throw Error(`you specify uploadRaw, but you also provide a folder key!`);
|
|
||||||
}
|
|
||||||
// folder
|
|
||||||
if (cipher.isPasswordEmpty() || cipher.isFolderAware()) {
|
|
||||||
// if not encrypted, || encrypted isFolderAware, mkdir a remote folder
|
|
||||||
if (foldersCreatedBefore?.has(uploadFile)) {
|
|
||||||
// created, pass
|
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
await retryReq(
|
|
||||||
() =>
|
|
||||||
client.dropbox.filesCreateFolderV2({
|
|
||||||
path: uploadFile,
|
|
||||||
}),
|
|
||||||
fileOrFolderPath
|
|
||||||
);
|
|
||||||
foldersCreatedBefore?.add(uploadFile);
|
|
||||||
} catch (e: unknown) {
|
|
||||||
const err = e as DropboxResponseError<files.CreateFolderError>;
|
|
||||||
if (err.status === undefined) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
if (err.status === 409) {
|
|
||||||
// pass
|
|
||||||
foldersCreatedBefore?.add(uploadFile);
|
|
||||||
} else {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const res = await getRemoteMeta(client, uploadFile);
|
|
||||||
return {
|
|
||||||
entity: res,
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
// if encrypted && !isFolderAware(),
|
|
||||||
// upload a fake file with the encrypted file name
|
|
||||||
await retryReq(
|
|
||||||
() =>
|
|
||||||
client.dropbox.filesUpload({
|
|
||||||
path: uploadFile,
|
|
||||||
contents: "",
|
|
||||||
client_modified: mtimeStr,
|
|
||||||
}),
|
|
||||||
fileOrFolderPath
|
|
||||||
);
|
);
|
||||||
return {
|
|
||||||
entity: await getRemoteMeta(client, uploadFile),
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// file
|
|
||||||
// we ignore isRecursively parameter here
|
|
||||||
let localContent = undefined;
|
|
||||||
if (uploadRaw) {
|
|
||||||
if (typeof rawContent === "string") {
|
|
||||||
localContent = new TextEncoder().encode(rawContent).buffer;
|
|
||||||
} else {
|
|
||||||
localContent = rawContent;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (vault === undefined) {
|
|
||||||
throw new Error(
|
|
||||||
`the vault variable is not passed but we want to read ${fileOrFolderPath} for Dropbox`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
localContent = await vault.adapter.readBinary(fileOrFolderPath);
|
|
||||||
}
|
|
||||||
let remoteContent = localContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteContent = await cipher.encryptContent(localContent);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const mtimeFixed = Math.floor(mtime / 1000.0) * 1000;
|
||||||
|
const ctimeFixed = Math.floor(ctime / 1000.0) * 1000;
|
||||||
|
const mtimeStr = new Date(mtimeFixed)
|
||||||
|
.toISOString()
|
||||||
|
.replace(/\.\d{3}Z$/, "Z");
|
||||||
|
|
||||||
// in dropbox, we don't need to create folders before uploading! cool!
|
// in dropbox, we don't need to create folders before uploading! cool!
|
||||||
// TODO: filesUploadSession for larger files (>=150 MB)
|
// TODO: filesUploadSession for larger files (>=150 MB)
|
||||||
|
|
||||||
await retryReq(
|
await retryReq(
|
||||||
() =>
|
() =>
|
||||||
client.dropbox.filesUpload({
|
this.dropbox.filesUpload({
|
||||||
path: uploadFile,
|
path: key,
|
||||||
contents: remoteContent,
|
contents: content,
|
||||||
mode: {
|
mode: {
|
||||||
".tag": "overwrite",
|
".tag": "overwrite",
|
||||||
},
|
},
|
||||||
client_modified: mtimeStr,
|
client_modified: mtimeStr,
|
||||||
}),
|
}),
|
||||||
fileOrFolderPath
|
origKey // hint
|
||||||
);
|
);
|
||||||
|
|
||||||
// we want to mark that parent folders are created
|
// we want to mark that parent folders are created
|
||||||
if (foldersCreatedBefore !== undefined) {
|
if (this.foldersCreatedBefore !== undefined) {
|
||||||
const dirs = getFolderLevels(uploadFile).map((x) =>
|
const dirs = getFolderLevels(origKey).map((x) =>
|
||||||
getDropboxPath(x, client.remoteBaseDir)
|
getDropboxPath(x, this.remoteBaseDir)
|
||||||
);
|
);
|
||||||
for (const dir of dirs) {
|
for (const dir of dirs) {
|
||||||
foldersCreatedBefore?.add(dir);
|
this.foldersCreatedBefore?.add(dir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return {
|
return await this._statFromRoot(key);
|
||||||
entity: await getRemoteMeta(client, uploadFile),
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
export const listAllFromRemote = async (client: WrappedDropboxClient) => {
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
await client.init();
|
await this._init();
|
||||||
let res = await client.dropbox.filesListFolder({
|
if (key.endsWith("/")) {
|
||||||
path: `/${client.remoteBaseDir}`,
|
throw new Error(`you should not call readFile on folder ${key}`);
|
||||||
recursive: true,
|
|
||||||
include_deleted: false,
|
|
||||||
limit: 1000,
|
|
||||||
});
|
|
||||||
if (res.status !== 200) {
|
|
||||||
throw Error(JSON.stringify(res));
|
|
||||||
}
|
|
||||||
// console.info(res);
|
|
||||||
|
|
||||||
const contents = res.result.entries;
|
|
||||||
const unifiedContents = contents
|
|
||||||
.filter((x) => x[".tag"] !== "deleted")
|
|
||||||
.filter((x) => x.path_display !== `/${client.remoteBaseDir}`)
|
|
||||||
.map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir));
|
|
||||||
|
|
||||||
while (res.result.has_more) {
|
|
||||||
res = await client.dropbox.filesListFolderContinue({
|
|
||||||
cursor: res.result.cursor,
|
|
||||||
});
|
|
||||||
if (res.status !== 200) {
|
|
||||||
throw Error(JSON.stringify(res));
|
|
||||||
}
|
}
|
||||||
|
const downloadFile = getDropboxPath(key, this.remoteBaseDir);
|
||||||
const contents2 = res.result.entries;
|
return await this._readFileFromRoot(downloadFile);
|
||||||
const unifiedContents2 = contents2
|
|
||||||
.filter((x) => x[".tag"] !== "deleted")
|
|
||||||
.filter((x) => x.path_display !== `/${client.remoteBaseDir}`)
|
|
||||||
.map((x) => fromDropboxItemToEntity(x, client.remoteBaseDir));
|
|
||||||
unifiedContents.push(...unifiedContents2);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fixEntityListCasesInplace(unifiedContents);
|
async _readFileFromRoot(key: string): Promise<ArrayBuffer> {
|
||||||
|
const rsp = await retryReq(
|
||||||
return unifiedContents;
|
|
||||||
};
|
|
||||||
|
|
||||||
const downloadFromRemoteRaw = async (
|
|
||||||
client: WrappedDropboxClient,
|
|
||||||
remotePath: string
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
const rsp = await retryReq(
|
|
||||||
() =>
|
|
||||||
client.dropbox.filesDownload({
|
|
||||||
path: remotePath,
|
|
||||||
}),
|
|
||||||
`downloadFromRemoteRaw=${remotePath}`
|
|
||||||
);
|
|
||||||
if (rsp === undefined) {
|
|
||||||
throw Error(`unknown rsp from dropbox download: ${rsp}`);
|
|
||||||
}
|
|
||||||
if ((rsp.result as any).fileBlob !== undefined) {
|
|
||||||
// we get a Blob
|
|
||||||
const content = (rsp.result as any).fileBlob as Blob;
|
|
||||||
return await content.arrayBuffer();
|
|
||||||
} else if ((rsp.result as any).fileBinary !== undefined) {
|
|
||||||
// we get a Buffer
|
|
||||||
const content = (rsp.result as any).fileBinary as Buffer;
|
|
||||||
return bufferToArrayBuffer(content);
|
|
||||||
} else {
|
|
||||||
throw Error(`unknown rsp from dropbox download: ${rsp}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const downloadFromRemote = async (
|
|
||||||
client: WrappedDropboxClient,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault,
|
|
||||||
mtime: number,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
skipSaving: boolean = false
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (!skipSaving) {
|
|
||||||
await mkdirpInVault(fileOrFolderPath, vault);
|
|
||||||
}
|
|
||||||
|
|
||||||
// the file is always local file
|
|
||||||
// we need to encrypt it
|
|
||||||
|
|
||||||
if (isFolder) {
|
|
||||||
// mkdirp locally is enough
|
|
||||||
// do nothing here
|
|
||||||
return new ArrayBuffer(0);
|
|
||||||
} else {
|
|
||||||
let downloadFile = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
downloadFile = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
downloadFile = getDropboxPath(downloadFile, client.remoteBaseDir);
|
|
||||||
const remoteContent = await downloadFromRemoteRaw(client, downloadFile);
|
|
||||||
let localContent = remoteContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
localContent = await cipher.decryptContent(remoteContent);
|
|
||||||
}
|
|
||||||
if (!skipSaving) {
|
|
||||||
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
|
|
||||||
mtime: mtime,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return localContent;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const deleteFromRemote = async (
|
|
||||||
client: WrappedDropboxClient,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = ""
|
|
||||||
) => {
|
|
||||||
if (fileOrFolderPath === "/") {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let remoteFileName = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteFileName = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
remoteFileName = getDropboxPath(remoteFileName, client.remoteBaseDir);
|
|
||||||
|
|
||||||
await client.init();
|
|
||||||
try {
|
|
||||||
await retryReq(
|
|
||||||
() =>
|
() =>
|
||||||
client.dropbox.filesDeleteV2({
|
this.dropbox.filesDownload({
|
||||||
path: remoteFileName,
|
path: key,
|
||||||
}),
|
}),
|
||||||
fileOrFolderPath
|
`downloadFromRemoteRaw=${key}`
|
||||||
);
|
);
|
||||||
} catch (err) {
|
if (rsp === undefined) {
|
||||||
console.error("some error while deleting");
|
throw Error(`unknown rsp from dropbox download: ${rsp}`);
|
||||||
console.error(err);
|
}
|
||||||
|
if ((rsp.result as any).fileBlob !== undefined) {
|
||||||
|
// we get a Blob
|
||||||
|
const content = (rsp.result as any).fileBlob as Blob;
|
||||||
|
return await content.arrayBuffer();
|
||||||
|
} else if ((rsp.result as any).fileBinary !== undefined) {
|
||||||
|
// we get a Buffer
|
||||||
|
const content = (rsp.result as any).fileBinary as Buffer;
|
||||||
|
return bufferToArrayBuffer(content);
|
||||||
|
} else {
|
||||||
|
throw Error(`unknown rsp from dropbox download: ${rsp}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
export const checkConnectivity = async (
|
async rm(key: string): Promise<void> {
|
||||||
client: WrappedDropboxClient,
|
if (key === "/") {
|
||||||
callbackFunc?: any
|
return;
|
||||||
) => {
|
}
|
||||||
try {
|
const remoteFileName = getDropboxPath(key, this.remoteBaseDir);
|
||||||
await client.init();
|
|
||||||
const results = await getRemoteMeta(client, `/${client.remoteBaseDir}`);
|
await this._init();
|
||||||
if (results === undefined) {
|
try {
|
||||||
|
await retryReq(
|
||||||
|
() =>
|
||||||
|
this.dropbox.filesDeleteV2({
|
||||||
|
path: remoteFileName,
|
||||||
|
}),
|
||||||
|
key // just a hint here
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
console.error("some error while deleting");
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await this._init();
|
||||||
|
const results = await this._statFromRoot(`/${this.remoteBaseDir}`);
|
||||||
|
if (results === undefined) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
console.debug(err);
|
||||||
|
callbackFunc?.(err);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
|
||||||
} catch (err) {
|
|
||||||
console.debug(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
export const getUserDisplayName = async (client: WrappedDropboxClient) => {
|
async getUserDisplayName() {
|
||||||
await client.init();
|
await this._init();
|
||||||
const acct = await client.dropbox.usersGetCurrentAccount();
|
const acct = await this.dropbox.usersGetCurrentAccount();
|
||||||
return acct.result.name.display_name;
|
return acct.result.name.display_name;
|
||||||
};
|
}
|
||||||
|
|
||||||
export const revokeAuth = async (client: WrappedDropboxClient) => {
|
async revokeAuth() {
|
||||||
await client.init();
|
try {
|
||||||
await client.dropbox.authTokenRevoke();
|
await this._init();
|
||||||
};
|
await this.dropbox.authTokenRevoke();
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,553 @@
|
||||||
|
import { CipherMethodType, Entity } from "./baseTypes";
|
||||||
|
import * as openssl from "./encryptOpenSSL";
|
||||||
|
import * as rclone from "./encryptRClone";
|
||||||
|
import { isVaildText } from "./misc";
|
||||||
|
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
|
import cloneDeep from "lodash/cloneDeep";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* quick guess, no actual decryption here
|
||||||
|
* @param name
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
function isLikelyOpenSSLEncryptedName(name: string): boolean {
|
||||||
|
if (
|
||||||
|
name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE32) ||
|
||||||
|
name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE64URL)
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* quick guess, no actual decryption here
|
||||||
|
* @param name
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
function isLikelyEncryptedName(name: string): boolean {
|
||||||
|
return isLikelyOpenSSLEncryptedName(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* quick guess, no actual decryption here, only openssl can be guessed here
|
||||||
|
* @param name
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
function isLikelyEncryptedNameNotMatchMethod(
|
||||||
|
name: string,
|
||||||
|
method: CipherMethodType
|
||||||
|
): boolean {
|
||||||
|
if (isLikelyOpenSSLEncryptedName(name) && method !== "openssl-base64") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (!isLikelyOpenSSLEncryptedName(name) && method === "openssl-base64") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PasswordCheckType {
|
||||||
|
ok: boolean;
|
||||||
|
reason:
|
||||||
|
| "empty_remote"
|
||||||
|
| "unknown_encryption_method"
|
||||||
|
| "remote_encrypted_local_no_password"
|
||||||
|
| "password_matched"
|
||||||
|
| "password_or_method_not_matched_or_remote_not_encrypted"
|
||||||
|
| "likely_no_password_both_sides"
|
||||||
|
| "encryption_method_not_matched";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Useful if isPasswordEmpty()
|
||||||
|
*/
|
||||||
|
function copyEntityAndCopyKeyEncSizeEnc(entity: Entity) {
|
||||||
|
const res = cloneDeep(entity);
|
||||||
|
res["keyEnc"] = res["keyRaw"];
|
||||||
|
res["sizeEnc"] = res["sizeRaw"];
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class FakeFsEncrypt extends FakeFs {
|
||||||
|
innerFs: FakeFs;
|
||||||
|
readonly password: string;
|
||||||
|
readonly method: CipherMethodType;
|
||||||
|
cipherRClone?: rclone.CipherRclone;
|
||||||
|
cacheMapOrigToEnc: Record<string, string>;
|
||||||
|
hasCacheMap: boolean;
|
||||||
|
kind: string;
|
||||||
|
innerWalkResultCache?: Entity[];
|
||||||
|
innerWalkResultCacheTime?: number;
|
||||||
|
|
||||||
|
constructor(innerFs: FakeFs, password: string, method: CipherMethodType) {
|
||||||
|
super();
|
||||||
|
this.innerFs = innerFs;
|
||||||
|
this.password = password ?? "";
|
||||||
|
this.method = method;
|
||||||
|
this.cacheMapOrigToEnc = {};
|
||||||
|
this.hasCacheMap = false;
|
||||||
|
|
||||||
|
this.kind = `encrypt(${this.innerFs.kind},${method})`;
|
||||||
|
|
||||||
|
if (method === "rclone-base64") {
|
||||||
|
this.cipherRClone = new rclone.CipherRclone(password, 5);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isPasswordEmpty() {
|
||||||
|
return this.password === "";
|
||||||
|
}
|
||||||
|
|
||||||
|
isFolderAware() {
|
||||||
|
if (this.method === "openssl-base64") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (this.method === "rclone-base64") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
throw Error(`no idea about isFolderAware for method=${this.method}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* we want a little caching here.
|
||||||
|
*/
|
||||||
|
async _getInnerWalkResult(): Promise<Entity[]> {
|
||||||
|
let innerWalkResult: Entity[] | undefined = undefined;
|
||||||
|
if (
|
||||||
|
this.innerWalkResultCacheTime !== undefined &&
|
||||||
|
this.innerWalkResultCacheTime >= Date.now() - 1000
|
||||||
|
) {
|
||||||
|
innerWalkResult = this.innerWalkResultCache!;
|
||||||
|
} else {
|
||||||
|
innerWalkResult = await this.innerFs.walk();
|
||||||
|
this.innerWalkResultCache = innerWalkResult;
|
||||||
|
this.innerWalkResultCacheTime = Date.now();
|
||||||
|
}
|
||||||
|
return innerWalkResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
async isPasswordOk(): Promise<PasswordCheckType> {
|
||||||
|
const innerWalkResult = await this._getInnerWalkResult();
|
||||||
|
|
||||||
|
if (innerWalkResult === undefined || innerWalkResult.length === 0) {
|
||||||
|
// remote empty
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
reason: "empty_remote",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const santyCheckKey = innerWalkResult[0].keyRaw;
|
||||||
|
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
// TODO: no way to distinguish remote rclone encrypted
|
||||||
|
// if local has no password??
|
||||||
|
if (isLikelyEncryptedName(santyCheckKey)) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
reason: "remote_encrypted_local_no_password",
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
reason: "likely_no_password_both_sides",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (this.method === "unknown") {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
reason: "unknown_encryption_method",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (isLikelyEncryptedNameNotMatchMethod(santyCheckKey, this.method)) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
reason: "encryption_method_not_matched",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const k = await this._decryptName(santyCheckKey);
|
||||||
|
if (k === undefined) {
|
||||||
|
throw Error(`decryption failed`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
reason: "password_matched",
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
reason: "password_or_method_not_matched_or_remote_not_encrypted",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async walk(): Promise<Entity[]> {
|
||||||
|
const innerWalkResult = await this._getInnerWalkResult();
|
||||||
|
|
||||||
|
const res: Entity[] = [];
|
||||||
|
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
for (const innerEntity of innerWalkResult) {
|
||||||
|
res.push(copyEntityAndCopyKeyEncSizeEnc(innerEntity));
|
||||||
|
this.cacheMapOrigToEnc[innerEntity.key!] = innerEntity.key!;
|
||||||
|
}
|
||||||
|
this.hasCacheMap = true;
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
for (const innerEntity of innerWalkResult) {
|
||||||
|
const key = await this._decryptName(innerEntity.keyRaw);
|
||||||
|
const size = key.endsWith("/") ? 0 : undefined;
|
||||||
|
res.push({
|
||||||
|
key: key,
|
||||||
|
keyRaw: innerEntity.keyRaw,
|
||||||
|
keyEnc: innerEntity.key!,
|
||||||
|
mtimeCli: innerEntity.mtimeCli,
|
||||||
|
mtimeSvr: innerEntity.mtimeSvr,
|
||||||
|
size: size,
|
||||||
|
sizeEnc: innerEntity.size!,
|
||||||
|
sizeRaw: innerEntity.sizeRaw,
|
||||||
|
hash: undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.cacheMapOrigToEnc[key] = innerEntity.keyRaw;
|
||||||
|
}
|
||||||
|
this.hasCacheMap = true;
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(key: string): Promise<Entity> {
|
||||||
|
if (!this.hasCacheMap) {
|
||||||
|
throw new Error("You have to build the cacheMap firstly for stat");
|
||||||
|
}
|
||||||
|
const keyEnc = this.cacheMapOrigToEnc[key];
|
||||||
|
if (keyEnc === undefined) {
|
||||||
|
throw new Error(`no encrypted key ${key} before!`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const innerEntity = await this.innerFs.stat(keyEnc);
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
return copyEntityAndCopyKeyEncSizeEnc(innerEntity);
|
||||||
|
} else {
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
keyRaw: innerEntity.keyRaw,
|
||||||
|
keyEnc: innerEntity.key!,
|
||||||
|
mtimeCli: innerEntity.mtimeCli,
|
||||||
|
mtimeSvr: innerEntity.mtimeSvr,
|
||||||
|
size: undefined,
|
||||||
|
sizeEnc: innerEntity.size!,
|
||||||
|
sizeRaw: innerEntity.sizeRaw,
|
||||||
|
hash: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity> {
|
||||||
|
if (!this.hasCacheMap) {
|
||||||
|
throw new Error("You have to build the cacheMap firstly for mkdir");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!key.endsWith("/")) {
|
||||||
|
throw new Error(`should not call mkdir on ${key}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let keyEnc = this.cacheMapOrigToEnc[key];
|
||||||
|
if (keyEnc === undefined) {
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
keyEnc = key;
|
||||||
|
} else {
|
||||||
|
keyEnc = await this._encryptName(key);
|
||||||
|
}
|
||||||
|
this.cacheMapOrigToEnc[key] = keyEnc;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.isPasswordEmpty() || this.isFolderAware()) {
|
||||||
|
const innerEntity = await this.innerFs.mkdir(keyEnc, mtime, ctime);
|
||||||
|
return copyEntityAndCopyKeyEncSizeEnc(innerEntity);
|
||||||
|
} else {
|
||||||
|
const now = Date.now();
|
||||||
|
const innerEntity = await this.innerFs.writeFile(
|
||||||
|
keyEnc,
|
||||||
|
new ArrayBuffer(0),
|
||||||
|
mtime ?? now,
|
||||||
|
ctime ?? now
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
keyRaw: innerEntity.keyRaw,
|
||||||
|
keyEnc: innerEntity.key!,
|
||||||
|
mtimeCli: innerEntity.mtimeCli,
|
||||||
|
mtimeSvr: innerEntity.mtimeSvr,
|
||||||
|
size: 0,
|
||||||
|
sizeEnc: innerEntity.size!,
|
||||||
|
sizeRaw: innerEntity.sizeRaw,
|
||||||
|
hash: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeFile(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (!this.hasCacheMap) {
|
||||||
|
throw new Error("You have to build the cacheMap firstly for readFile");
|
||||||
|
}
|
||||||
|
let keyEnc = this.cacheMapOrigToEnc[key];
|
||||||
|
if (keyEnc === undefined) {
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
keyEnc = key;
|
||||||
|
} else {
|
||||||
|
keyEnc = await this._encryptName(key);
|
||||||
|
}
|
||||||
|
this.cacheMapOrigToEnc[key] = keyEnc;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
const innerEntity = await this.innerFs.writeFile(
|
||||||
|
keyEnc,
|
||||||
|
content,
|
||||||
|
mtime,
|
||||||
|
ctime
|
||||||
|
);
|
||||||
|
return copyEntityAndCopyKeyEncSizeEnc(innerEntity);
|
||||||
|
} else {
|
||||||
|
const contentEnc = await this._encryptContent(content);
|
||||||
|
const innerEntity = await this.innerFs.writeFile(
|
||||||
|
keyEnc,
|
||||||
|
contentEnc,
|
||||||
|
mtime,
|
||||||
|
ctime
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
keyRaw: innerEntity.keyRaw,
|
||||||
|
keyEnc: innerEntity.key!,
|
||||||
|
mtimeCli: innerEntity.mtimeCli,
|
||||||
|
mtimeSvr: innerEntity.mtimeSvr,
|
||||||
|
size: undefined,
|
||||||
|
sizeEnc: innerEntity.size!,
|
||||||
|
sizeRaw: innerEntity.sizeRaw,
|
||||||
|
hash: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
|
if (!this.hasCacheMap) {
|
||||||
|
throw new Error("You have to build the cacheMap firstly for readFile");
|
||||||
|
}
|
||||||
|
const keyEnc = this.cacheMapOrigToEnc[key];
|
||||||
|
if (keyEnc === undefined) {
|
||||||
|
throw new Error(`no encrypted key ${key} before! cannot readFile`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentEnc = await this.innerFs.readFile(keyEnc);
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
return contentEnc;
|
||||||
|
} else {
|
||||||
|
const res = await this._decryptContent(contentEnc);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async rm(key: string): Promise<void> {
|
||||||
|
if (!this.hasCacheMap) {
|
||||||
|
throw new Error("You have to build the cacheMap firstly for rm");
|
||||||
|
}
|
||||||
|
const keyEnc = this.cacheMapOrigToEnc[key];
|
||||||
|
if (keyEnc === undefined) {
|
||||||
|
throw new Error(`no encrypted key ${key} before! cannot rm`);
|
||||||
|
}
|
||||||
|
return await this.innerFs.rm(keyEnc);
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
|
return await this.innerFs.checkConnect(callbackFunc);
|
||||||
|
}
|
||||||
|
|
||||||
|
async closeResources() {
|
||||||
|
if (this.method === "rclone-base64" && this.cipherRClone !== undefined) {
|
||||||
|
this.cipherRClone.closeResources();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async encryptEntity(input: Entity): Promise<Entity> {
|
||||||
|
if (input.key === undefined) {
|
||||||
|
// input.key should always have value
|
||||||
|
throw Error(`input ${input.keyRaw} is abnormal without key`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.isPasswordEmpty()) {
|
||||||
|
return copyEntityAndCopyKeyEncSizeEnc(input);
|
||||||
|
}
|
||||||
|
|
||||||
|
// below is for having password
|
||||||
|
const local = cloneDeep(input);
|
||||||
|
if (local.sizeEnc === undefined && local.size !== undefined) {
|
||||||
|
// it's not filled yet, we fill it
|
||||||
|
// local.size is possibly undefined if it's "prevSync" Entity
|
||||||
|
// but local.key should always have value
|
||||||
|
local.sizeEnc = this._getSizeFromOrigToEnc(local.size);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (local.keyEnc === undefined || local.keyEnc === "") {
|
||||||
|
let keyEnc = this.cacheMapOrigToEnc[input.key];
|
||||||
|
if (keyEnc !== undefined && keyEnc !== "" && keyEnc !== local.key) {
|
||||||
|
// we can reuse remote encrypted key if any
|
||||||
|
local.keyEnc = keyEnc;
|
||||||
|
} else {
|
||||||
|
// we assign a new encrypted key because of no remote
|
||||||
|
keyEnc = await this._encryptName(input.key);
|
||||||
|
local.keyEnc = keyEnc;
|
||||||
|
// remember to add back to cache!
|
||||||
|
this.cacheMapOrigToEnc[input.key] = keyEnc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return local;
|
||||||
|
}
|
||||||
|
|
||||||
|
async _encryptContent(content: ArrayBuffer) {
|
||||||
|
// console.debug("start encryptContent");
|
||||||
|
if (this.password === "") {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
if (this.method === "openssl-base64") {
|
||||||
|
const res = await openssl.encryptArrayBuffer(content, this.password);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot encrypt content`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else if (this.method === "rclone-base64") {
|
||||||
|
const res =
|
||||||
|
await this.cipherRClone!.encryptContentByCallingWorker(content);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot encrypt content`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
throw Error(`not supported encrypt method=${this.method}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async _decryptContent(content: ArrayBuffer) {
|
||||||
|
// console.debug("start decryptContent");
|
||||||
|
if (this.password === "") {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
if (this.method === "openssl-base64") {
|
||||||
|
const res = await openssl.decryptArrayBuffer(content, this.password);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot decrypt content`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else if (this.method === "rclone-base64") {
|
||||||
|
const res =
|
||||||
|
await this.cipherRClone!.decryptContentByCallingWorker(content);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot decrypt content`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
throw Error(`not supported decrypt method=${this.method}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async _encryptName(name: string) {
|
||||||
|
// console.debug("start encryptName");
|
||||||
|
if (this.password === "") {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
if (this.method === "openssl-base64") {
|
||||||
|
const res = await openssl.encryptStringToBase64url(name, this.password);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot encrypt name=${name}`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else if (this.method === "rclone-base64") {
|
||||||
|
const res = await this.cipherRClone!.encryptNameByCallingWorker(name);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot encrypt name=${name}`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
throw Error(`not supported encrypt method=${this.method}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async _decryptName(name: string): Promise<string> {
|
||||||
|
// console.debug("start decryptName");
|
||||||
|
if (this.password === "") {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
if (this.method === "openssl-base64") {
|
||||||
|
if (name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
||||||
|
// backward compitable with the openssl-base32
|
||||||
|
try {
|
||||||
|
const res = await openssl.decryptBase32ToString(name, this.password);
|
||||||
|
if (res !== undefined && isVaildText(res)) {
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
throw Error(`cannot decrypt name=${name}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
throw Error(`cannot decrypt name=${name}`);
|
||||||
|
}
|
||||||
|
} else if (name.startsWith(openssl.MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
|
||||||
|
try {
|
||||||
|
const res = await openssl.decryptBase64urlToString(
|
||||||
|
name,
|
||||||
|
this.password
|
||||||
|
);
|
||||||
|
if (res !== undefined && isVaildText(res)) {
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
throw Error(`cannot decrypt name=${name}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
throw Error(`cannot decrypt name=${name}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw Error(
|
||||||
|
`method=${this.method} but the name=${name}, likely mismatch`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else if (this.method === "rclone-base64") {
|
||||||
|
const res = await this.cipherRClone!.decryptNameByCallingWorker(name);
|
||||||
|
if (res === undefined) {
|
||||||
|
throw Error(`cannot decrypt name=${name}`);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
} else {
|
||||||
|
throw Error(`not supported decrypt method=${this.method}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_getSizeFromOrigToEnc(x: number) {
|
||||||
|
if (this.password === "") {
|
||||||
|
return x;
|
||||||
|
}
|
||||||
|
if (this.method === "openssl-base64") {
|
||||||
|
return openssl.getSizeFromOrigToEnc(x);
|
||||||
|
} else if (this.method === "rclone-base64") {
|
||||||
|
return rclone.getSizeFromOrigToEnc(x);
|
||||||
|
} else {
|
||||||
|
throw Error(`not supported encrypt method=${this.method}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserDisplayName(): Promise<string> {
|
||||||
|
return await this.innerFs.getUserDisplayName();
|
||||||
|
}
|
||||||
|
|
||||||
|
async revokeAuth(): Promise<any> {
|
||||||
|
return await this.innerFs.revokeAuth();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,45 @@
|
||||||
|
import { RemotelySavePluginSettings } from "./baseTypes";
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
|
import { FakeFsDropbox } from "./fsDropbox";
|
||||||
|
import { FakeFsOnedrive } from "./fsOnedrive";
|
||||||
|
import { FakeFsS3 } from "./fsS3";
|
||||||
|
import { FakeFsWebdav } from "./fsWebdav";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To avoid circular dependency, we need a new file here.
|
||||||
|
*/
|
||||||
|
export function getClient(
|
||||||
|
settings: RemotelySavePluginSettings,
|
||||||
|
vaultName: string,
|
||||||
|
saveUpdatedConfigFunc: () => Promise<any>
|
||||||
|
): FakeFs {
|
||||||
|
switch (settings.serviceType) {
|
||||||
|
case "s3":
|
||||||
|
return new FakeFsS3(settings.s3);
|
||||||
|
break;
|
||||||
|
case "webdav":
|
||||||
|
return new FakeFsWebdav(
|
||||||
|
settings.webdav,
|
||||||
|
vaultName,
|
||||||
|
saveUpdatedConfigFunc
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case "dropbox":
|
||||||
|
return new FakeFsDropbox(
|
||||||
|
settings.dropbox,
|
||||||
|
vaultName,
|
||||||
|
saveUpdatedConfigFunc
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case "onedrive":
|
||||||
|
return new FakeFsOnedrive(
|
||||||
|
settings.onedrive,
|
||||||
|
vaultName,
|
||||||
|
saveUpdatedConfigFunc
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw Error(`cannot init client for serviceType=${settings.serviceType}`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,171 @@
|
||||||
|
import { DEFAULT_DEBUG_FOLDER, Entity } from "./baseTypes";
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
|
|
||||||
|
import { TFile, TFolder, type Vault } from "obsidian";
|
||||||
|
import { listFilesInObsFolder } from "./obsFolderLister";
|
||||||
|
import { Profiler } from "./profiler";
|
||||||
|
import { getFolderLevels, mkdirpInVault, statFix } from "./misc";
|
||||||
|
|
||||||
|
export class FakeFsLocal extends FakeFs {
|
||||||
|
vault: Vault;
|
||||||
|
syncConfigDir: boolean;
|
||||||
|
configDir: string;
|
||||||
|
pluginID: string;
|
||||||
|
profiler: Profiler;
|
||||||
|
deleteToWhere: "obsidian" | "system";
|
||||||
|
kind: "local";
|
||||||
|
constructor(
|
||||||
|
vault: Vault,
|
||||||
|
syncConfigDir: boolean,
|
||||||
|
configDir: string,
|
||||||
|
pluginID: string,
|
||||||
|
profiler: Profiler,
|
||||||
|
deleteToWhere: "obsidian" | "system"
|
||||||
|
) {
|
||||||
|
super();
|
||||||
|
|
||||||
|
this.vault = vault;
|
||||||
|
this.syncConfigDir = syncConfigDir;
|
||||||
|
this.configDir = configDir;
|
||||||
|
this.pluginID = pluginID;
|
||||||
|
this.profiler = profiler;
|
||||||
|
this.deleteToWhere = deleteToWhere;
|
||||||
|
this.kind = "local";
|
||||||
|
}
|
||||||
|
|
||||||
|
async walk(): Promise<Entity[]> {
|
||||||
|
this.profiler.addIndent();
|
||||||
|
this.profiler.insert("enter walk for local");
|
||||||
|
const local: Entity[] = [];
|
||||||
|
|
||||||
|
const localTAbstractFiles = this.vault.getAllLoadedFiles();
|
||||||
|
this.profiler.insert("finish getting walk for local");
|
||||||
|
for (const entry of localTAbstractFiles) {
|
||||||
|
let r: Entity | undefined = undefined;
|
||||||
|
let key = entry.path;
|
||||||
|
|
||||||
|
if (entry.path === "/") {
|
||||||
|
// ignore
|
||||||
|
continue;
|
||||||
|
} else if (entry instanceof TFile) {
|
||||||
|
let mtimeLocal: number | undefined = entry.stat.mtime;
|
||||||
|
if (mtimeLocal <= 0) {
|
||||||
|
mtimeLocal = entry.stat.ctime;
|
||||||
|
}
|
||||||
|
if (mtimeLocal === 0) {
|
||||||
|
mtimeLocal = undefined;
|
||||||
|
}
|
||||||
|
if (mtimeLocal === undefined) {
|
||||||
|
throw Error(
|
||||||
|
`Your file has last modified time 0: ${key}, don't know how to deal with it`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
r = {
|
||||||
|
key: entry.path, // local always unencrypted
|
||||||
|
keyRaw: entry.path,
|
||||||
|
mtimeCli: mtimeLocal,
|
||||||
|
mtimeSvr: mtimeLocal,
|
||||||
|
size: entry.stat.size, // local always unencrypted
|
||||||
|
sizeRaw: entry.stat.size,
|
||||||
|
};
|
||||||
|
} else if (entry instanceof TFolder) {
|
||||||
|
key = `${entry.path}/`;
|
||||||
|
r = {
|
||||||
|
key: key,
|
||||||
|
keyRaw: key,
|
||||||
|
size: 0,
|
||||||
|
sizeRaw: 0,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
throw Error(`unexpected ${entry}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (r.keyRaw.startsWith(DEFAULT_DEBUG_FOLDER)) {
|
||||||
|
// skip listing the debug folder,
|
||||||
|
// which should always not involved in sync
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
local.push(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.profiler.insert("finish transforming walk for local");
|
||||||
|
|
||||||
|
if (this.syncConfigDir) {
|
||||||
|
this.profiler.insert("into syncConfigDir");
|
||||||
|
const syncFiles = await listFilesInObsFolder(
|
||||||
|
this.configDir,
|
||||||
|
this.vault,
|
||||||
|
this.pluginID
|
||||||
|
);
|
||||||
|
for (const f of syncFiles) {
|
||||||
|
local.push(f);
|
||||||
|
}
|
||||||
|
this.profiler.insert("finish syncConfigDir");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.profiler.insert("finish walk for local");
|
||||||
|
this.profiler.removeIndent();
|
||||||
|
return local;
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(key: string): Promise<Entity> {
|
||||||
|
const statRes = await statFix(this.vault, key);
|
||||||
|
if (statRes === undefined || statRes === null) {
|
||||||
|
throw Error(`${key} does not exist! cannot stat for local`);
|
||||||
|
}
|
||||||
|
const isFolder = statRes.type === "folder";
|
||||||
|
return {
|
||||||
|
key: isFolder ? `${key}/` : key, // local always unencrypted
|
||||||
|
keyRaw: isFolder ? `${key}/` : key,
|
||||||
|
mtimeCli: statRes.mtime,
|
||||||
|
mtimeSvr: statRes.mtime,
|
||||||
|
size: statRes.size, // local always unencrypted
|
||||||
|
sizeRaw: statRes.size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity> {
|
||||||
|
// console.debug(`mkdir: ${key}`);
|
||||||
|
await mkdirpInVault(key, this.vault);
|
||||||
|
return await this.stat(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeFile(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
await this.vault.adapter.writeBinary(key, content, {
|
||||||
|
mtime: mtime,
|
||||||
|
});
|
||||||
|
return await this.stat(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
|
return await this.vault.adapter.readBinary(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async rm(key: string): Promise<void> {
|
||||||
|
if (this.deleteToWhere === "obsidian") {
|
||||||
|
await this.vault.adapter.trashLocal(key);
|
||||||
|
} else {
|
||||||
|
// "system"
|
||||||
|
if (!(await this.vault.adapter.trashSystem(key))) {
|
||||||
|
await this.vault.adapter.trashLocal(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserDisplayName(): Promise<string> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async revokeAuth(): Promise<any> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,52 @@
|
||||||
|
import { Entity } from "./baseTypes";
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
|
|
||||||
|
export class FakeFsMock extends FakeFs {
|
||||||
|
kind: "mock";
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super();
|
||||||
|
this.kind = "mock";
|
||||||
|
}
|
||||||
|
|
||||||
|
async walk(): Promise<Entity[]> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(key: string): Promise<Entity> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async mkdir(key: string, mtime: number, ctime: number): Promise<Entity> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeFile(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async rm(key: string): Promise<void> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserDisplayName(): Promise<string> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async revokeAuth(): Promise<any> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,29 +1,23 @@
|
||||||
import { CryptoProvider, PublicClientApplication } from "@azure/msal-node";
|
|
||||||
import { AuthenticationProvider } from "@microsoft/microsoft-graph-client";
|
|
||||||
import type {
|
import type {
|
||||||
DriveItem,
|
DriveItem,
|
||||||
FileSystemInfo,
|
FileSystemInfo,
|
||||||
UploadSession,
|
UploadSession,
|
||||||
User,
|
User,
|
||||||
} from "@microsoft/microsoft-graph-types";
|
} from "@microsoft/microsoft-graph-types";
|
||||||
|
import { CryptoProvider, PublicClientApplication } from "@azure/msal-node";
|
||||||
|
import { AuthenticationProvider } from "@microsoft/microsoft-graph-client";
|
||||||
import cloneDeep from "lodash/cloneDeep";
|
import cloneDeep from "lodash/cloneDeep";
|
||||||
import { request, requestUrl, requireApiVersion, Vault } from "obsidian";
|
import { request, requestUrl } from "obsidian";
|
||||||
import {
|
import {
|
||||||
VALID_REQURL,
|
|
||||||
COMMAND_CALLBACK_ONEDRIVE,
|
COMMAND_CALLBACK_ONEDRIVE,
|
||||||
DEFAULT_CONTENT_TYPE,
|
DEFAULT_CONTENT_TYPE,
|
||||||
|
Entity,
|
||||||
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
|
OAUTH2_FORCE_EXPIRE_MILLISECONDS,
|
||||||
OnedriveConfig,
|
OnedriveConfig,
|
||||||
Entity,
|
VALID_REQURL,
|
||||||
UploadedType,
|
|
||||||
} from "./baseTypes";
|
} from "./baseTypes";
|
||||||
import {
|
import { FakeFs } from "./fsAll";
|
||||||
bufferToArrayBuffer,
|
import { bufferToArrayBuffer } from "./misc";
|
||||||
getRandomArrayBuffer,
|
|
||||||
getRandomIntInclusive,
|
|
||||||
mkdirpInVault,
|
|
||||||
} from "./misc";
|
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
|
|
||||||
const SCOPES = ["User.Read", "Files.ReadWrite.AppFolder", "offline_access"];
|
const SCOPES = ["User.Read", "Files.ReadWrite.AppFolder", "offline_access"];
|
||||||
const REDIRECT_URI = `obsidian://${COMMAND_CALLBACK_ONEDRIVE}`;
|
const REDIRECT_URI = `obsidian://${COMMAND_CALLBACK_ONEDRIVE}`;
|
||||||
|
@ -237,23 +231,6 @@ const getOnedrivePath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
||||||
return key;
|
return key;
|
||||||
};
|
};
|
||||||
|
|
||||||
const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
|
||||||
const prefix = `/drive/special/approot:/${remoteBaseDir}`;
|
|
||||||
|
|
||||||
if (
|
|
||||||
!(fileOrFolderPath === prefix || fileOrFolderPath.startsWith(`${prefix}/`))
|
|
||||||
) {
|
|
||||||
throw Error(
|
|
||||||
`"${fileOrFolderPath}" doesn't starts with "${prefix}/" or equals to "${prefix}"`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fileOrFolderPath === prefix) {
|
|
||||||
return "/";
|
|
||||||
}
|
|
||||||
return fileOrFolderPath.slice(`${prefix}/`.length);
|
|
||||||
};
|
|
||||||
|
|
||||||
const constructFromDriveItemToEntityError = (x: DriveItem) => {
|
const constructFromDriveItemToEntityError = (x: DriveItem) => {
|
||||||
return `parentPath="${
|
return `parentPath="${
|
||||||
x.parentReference?.path ?? "(no parentReference or path)"
|
x.parentReference?.path ?? "(no parentReference or path)"
|
||||||
|
@ -361,15 +338,20 @@ const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => {
|
||||||
const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
|
const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
|
||||||
const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
|
const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
|
||||||
return {
|
return {
|
||||||
|
key: key,
|
||||||
keyRaw: key,
|
keyRaw: key,
|
||||||
mtimeSvr: mtimeSvr,
|
mtimeSvr: mtimeSvr,
|
||||||
mtimeCli: mtimeCli,
|
mtimeCli: mtimeCli,
|
||||||
|
size: isFolder ? 0 : x.size!,
|
||||||
sizeRaw: isFolder ? 0 : x.size!,
|
sizeRaw: isFolder ? 0 : x.size!,
|
||||||
// hash: ?? // TODO
|
// hash: ?? // TODO
|
||||||
etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// The client.
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
// to adapt to the required interface
|
// to adapt to the required interface
|
||||||
class MyAuthProvider implements AuthenticationProvider {
|
class MyAuthProvider implements AuthenticationProvider {
|
||||||
onedriveConfig: OnedriveConfig;
|
onedriveConfig: OnedriveConfig;
|
||||||
|
@ -381,7 +363,8 @@ class MyAuthProvider implements AuthenticationProvider {
|
||||||
this.onedriveConfig = onedriveConfig;
|
this.onedriveConfig = onedriveConfig;
|
||||||
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
||||||
}
|
}
|
||||||
getAccessToken = async () => {
|
|
||||||
|
async getAccessToken() {
|
||||||
if (
|
if (
|
||||||
this.onedriveConfig.accessToken === "" ||
|
this.onedriveConfig.accessToken === "" ||
|
||||||
this.onedriveConfig.refreshToken === ""
|
this.onedriveConfig.refreshToken === ""
|
||||||
|
@ -415,7 +398,7 @@ class MyAuthProvider implements AuthenticationProvider {
|
||||||
console.info("Onedrive accessToken updated");
|
console.info("Onedrive accessToken updated");
|
||||||
return this.onedriveConfig.accessToken;
|
return this.onedriveConfig.accessToken;
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -431,25 +414,31 @@ export const getShrinkedSettings = (onedriveConfig: OnedriveConfig) => {
|
||||||
return config;
|
return config;
|
||||||
};
|
};
|
||||||
|
|
||||||
export class WrappedOnedriveClient {
|
export class FakeFsOnedrive extends FakeFs {
|
||||||
|
kind: "onedrive";
|
||||||
onedriveConfig: OnedriveConfig;
|
onedriveConfig: OnedriveConfig;
|
||||||
remoteBaseDir: string;
|
remoteBaseDir: string;
|
||||||
vaultFolderExists: boolean;
|
vaultFolderExists: boolean;
|
||||||
authGetter: MyAuthProvider;
|
authGetter: MyAuthProvider;
|
||||||
saveUpdatedConfigFunc: () => Promise<any>;
|
saveUpdatedConfigFunc: () => Promise<any>;
|
||||||
|
foldersCreatedBefore: Set<string>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
onedriveConfig: OnedriveConfig,
|
onedriveConfig: OnedriveConfig,
|
||||||
remoteBaseDir: string,
|
vaultName: string,
|
||||||
saveUpdatedConfigFunc: () => Promise<any>
|
saveUpdatedConfigFunc: () => Promise<any>
|
||||||
) {
|
) {
|
||||||
|
super();
|
||||||
|
this.kind = "onedrive";
|
||||||
this.onedriveConfig = onedriveConfig;
|
this.onedriveConfig = onedriveConfig;
|
||||||
this.remoteBaseDir = remoteBaseDir;
|
this.remoteBaseDir = this.onedriveConfig.remoteBaseDir || vaultName || "";
|
||||||
this.vaultFolderExists = false;
|
this.vaultFolderExists = false;
|
||||||
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
||||||
this.authGetter = new MyAuthProvider(onedriveConfig, saveUpdatedConfigFunc);
|
this.authGetter = new MyAuthProvider(onedriveConfig, saveUpdatedConfigFunc);
|
||||||
|
this.foldersCreatedBefore = new Set();
|
||||||
}
|
}
|
||||||
|
|
||||||
init = async () => {
|
async _init() {
|
||||||
// check token
|
// check token
|
||||||
if (
|
if (
|
||||||
this.onedriveConfig.accessToken === "" ||
|
this.onedriveConfig.accessToken === "" ||
|
||||||
|
@ -463,14 +452,14 @@ export class WrappedOnedriveClient {
|
||||||
if (this.vaultFolderExists) {
|
if (this.vaultFolderExists) {
|
||||||
// console.info(`already checked, /${this.remoteBaseDir} exist before`)
|
// console.info(`already checked, /${this.remoteBaseDir} exist before`)
|
||||||
} else {
|
} else {
|
||||||
const k = await this.getJson("/drive/special/approot/children");
|
const k = await this._getJson("/drive/special/approot/children");
|
||||||
// console.debug(k);
|
// console.debug(k);
|
||||||
this.vaultFolderExists =
|
this.vaultFolderExists =
|
||||||
(k.value as DriveItem[]).filter((x) => x.name === this.remoteBaseDir)
|
(k.value as DriveItem[]).filter((x) => x.name === this.remoteBaseDir)
|
||||||
.length > 0;
|
.length > 0;
|
||||||
if (!this.vaultFolderExists) {
|
if (!this.vaultFolderExists) {
|
||||||
console.info(`remote does not have folder /${this.remoteBaseDir}`);
|
console.info(`remote does not have folder /${this.remoteBaseDir}`);
|
||||||
await this.postJson("/drive/special/approot/children", {
|
await this._postJson("/drive/special/approot/children", {
|
||||||
name: `${this.remoteBaseDir}`,
|
name: `${this.remoteBaseDir}`,
|
||||||
folder: {},
|
folder: {},
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
"@microsoft.graph.conflictBehavior": "replace",
|
||||||
|
@ -481,9 +470,9 @@ export class WrappedOnedriveClient {
|
||||||
// console.info(`remote folder /${this.remoteBaseDir} exists`);
|
// console.info(`remote folder /${this.remoteBaseDir} exists`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
buildUrl = (pathFragOrig: string) => {
|
_buildUrl(pathFragOrig: string) {
|
||||||
const API_PREFIX = "https://graph.microsoft.com/v1.0";
|
const API_PREFIX = "https://graph.microsoft.com/v1.0";
|
||||||
let theUrl = "";
|
let theUrl = "";
|
||||||
if (
|
if (
|
||||||
|
@ -501,10 +490,10 @@ export class WrappedOnedriveClient {
|
||||||
theUrl = theUrl.replace(/#/g, "%23");
|
theUrl = theUrl.replace(/#/g, "%23");
|
||||||
// console.debug(`building url: [${pathFragOrig}] => [${theUrl}]`)
|
// console.debug(`building url: [${pathFragOrig}] => [${theUrl}]`)
|
||||||
return theUrl;
|
return theUrl;
|
||||||
};
|
}
|
||||||
|
|
||||||
getJson = async (pathFragOrig: string) => {
|
async _getJson(pathFragOrig: string) {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this._buildUrl(pathFragOrig);
|
||||||
console.debug(`getJson, theUrl=${theUrl}`);
|
console.debug(`getJson, theUrl=${theUrl}`);
|
||||||
return JSON.parse(
|
return JSON.parse(
|
||||||
await request({
|
await request({
|
||||||
|
@ -517,10 +506,10 @@ export class WrappedOnedriveClient {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
};
|
}
|
||||||
|
|
||||||
postJson = async (pathFragOrig: string, payload: any) => {
|
async _postJson(pathFragOrig: string, payload: any) {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this._buildUrl(pathFragOrig);
|
||||||
console.debug(`postJson, theUrl=${theUrl}`);
|
console.debug(`postJson, theUrl=${theUrl}`);
|
||||||
return JSON.parse(
|
return JSON.parse(
|
||||||
await request({
|
await request({
|
||||||
|
@ -533,10 +522,10 @@ export class WrappedOnedriveClient {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
};
|
}
|
||||||
|
|
||||||
patchJson = async (pathFragOrig: string, payload: any) => {
|
async _patchJson(pathFragOrig: string, payload: any) {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this._buildUrl(pathFragOrig);
|
||||||
console.debug(`patchJson, theUrl=${theUrl}`);
|
console.debug(`patchJson, theUrl=${theUrl}`);
|
||||||
return JSON.parse(
|
return JSON.parse(
|
||||||
await request({
|
await request({
|
||||||
|
@ -549,10 +538,10 @@ export class WrappedOnedriveClient {
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
};
|
}
|
||||||
|
|
||||||
deleteJson = async (pathFragOrig: string) => {
|
async _deleteJson(pathFragOrig: string) {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this._buildUrl(pathFragOrig);
|
||||||
console.debug(`deleteJson, theUrl=${theUrl}`);
|
console.debug(`deleteJson, theUrl=${theUrl}`);
|
||||||
if (VALID_REQURL) {
|
if (VALID_REQURL) {
|
||||||
await requestUrl({
|
await requestUrl({
|
||||||
|
@ -570,10 +559,10 @@ export class WrappedOnedriveClient {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
putArrayBuffer = async (pathFragOrig: string, payload: ArrayBuffer) => {
|
async _putArrayBuffer(pathFragOrig: string, payload: ArrayBuffer) {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this._buildUrl(pathFragOrig);
|
||||||
console.debug(`putArrayBuffer, theUrl=${theUrl}`);
|
console.debug(`putArrayBuffer, theUrl=${theUrl}`);
|
||||||
// TODO:
|
// TODO:
|
||||||
// 20220401: On Android, requestUrl has issue that text becomes base64.
|
// 20220401: On Android, requestUrl has issue that text becomes base64.
|
||||||
|
@ -601,7 +590,7 @@ export class WrappedOnedriveClient {
|
||||||
});
|
});
|
||||||
return (await res.json()) as DriveItem | UploadSession;
|
return (await res.json()) as DriveItem | UploadSession;
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A specialized function to upload large files by parts
|
* A specialized function to upload large files by parts
|
||||||
|
@ -611,14 +600,14 @@ export class WrappedOnedriveClient {
|
||||||
* @param rangeEnd the end, exclusive
|
* @param rangeEnd the end, exclusive
|
||||||
* @param size
|
* @param size
|
||||||
*/
|
*/
|
||||||
putUint8ArrayByRange = async (
|
async _putUint8ArrayByRange(
|
||||||
pathFragOrig: string,
|
pathFragOrig: string,
|
||||||
payload: Uint8Array,
|
payload: Uint8Array,
|
||||||
rangeStart: number,
|
rangeStart: number,
|
||||||
rangeEnd: number,
|
rangeEnd: number,
|
||||||
size: number
|
size: number
|
||||||
) => {
|
) {
|
||||||
const theUrl = this.buildUrl(pathFragOrig);
|
const theUrl = this._buildUrl(pathFragOrig);
|
||||||
console.debug(
|
console.debug(
|
||||||
`putUint8ArrayByRange, theUrl=${theUrl}, range=${rangeStart}-${
|
`putUint8ArrayByRange, theUrl=${theUrl}, range=${rangeStart}-${
|
||||||
rangeEnd - 1
|
rangeEnd - 1
|
||||||
|
@ -654,201 +643,140 @@ export class WrappedOnedriveClient {
|
||||||
});
|
});
|
||||||
return (await res.json()) as DriveItem | UploadSession;
|
return (await res.json()) as DriveItem | UploadSession;
|
||||||
}
|
}
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export const getOnedriveClient = (
|
|
||||||
onedriveConfig: OnedriveConfig,
|
|
||||||
remoteBaseDir: string,
|
|
||||||
saveUpdatedConfigFunc: () => Promise<any>
|
|
||||||
) => {
|
|
||||||
return new WrappedOnedriveClient(
|
|
||||||
onedriveConfig,
|
|
||||||
remoteBaseDir,
|
|
||||||
saveUpdatedConfigFunc
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Use delta api to list all files and folders
|
|
||||||
* https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta?view=odsp-graph-online
|
|
||||||
* @param client
|
|
||||||
*/
|
|
||||||
export const listAllFromRemote = async (client: WrappedOnedriveClient) => {
|
|
||||||
await client.init();
|
|
||||||
|
|
||||||
const NEXT_LINK_KEY = "@odata.nextLink";
|
|
||||||
const DELTA_LINK_KEY = "@odata.deltaLink";
|
|
||||||
|
|
||||||
let res = await client.getJson(
|
|
||||||
`/drive/special/approot:/${client.remoteBaseDir}:/delta`
|
|
||||||
);
|
|
||||||
let driveItems = res.value as DriveItem[];
|
|
||||||
// console.debug(driveItems);
|
|
||||||
|
|
||||||
while (NEXT_LINK_KEY in res) {
|
|
||||||
res = await client.getJson(res[NEXT_LINK_KEY]);
|
|
||||||
driveItems.push(...cloneDeep(res.value as DriveItem[]));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lastly we should have delta link?
|
/**
|
||||||
if (DELTA_LINK_KEY in res) {
|
* Use delta api to list all files and folders
|
||||||
client.onedriveConfig.deltaLink = res[DELTA_LINK_KEY];
|
* https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_delta?view=odsp-graph-online
|
||||||
await client.saveUpdatedConfigFunc();
|
*/
|
||||||
|
async walk(): Promise<Entity[]> {
|
||||||
|
await this._init();
|
||||||
|
|
||||||
|
const NEXT_LINK_KEY = "@odata.nextLink";
|
||||||
|
const DELTA_LINK_KEY = "@odata.deltaLink";
|
||||||
|
|
||||||
|
let res = await this._getJson(
|
||||||
|
`/drive/special/approot:/${this.remoteBaseDir}:/delta`
|
||||||
|
);
|
||||||
|
let driveItems = res.value as DriveItem[];
|
||||||
|
// console.debug(driveItems);
|
||||||
|
|
||||||
|
while (NEXT_LINK_KEY in res) {
|
||||||
|
res = await this._getJson(res[NEXT_LINK_KEY]);
|
||||||
|
driveItems.push(...cloneDeep(res.value as DriveItem[]));
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastly we should have delta link?
|
||||||
|
if (DELTA_LINK_KEY in res) {
|
||||||
|
this.onedriveConfig.deltaLink = res[DELTA_LINK_KEY];
|
||||||
|
await this.saveUpdatedConfigFunc();
|
||||||
|
}
|
||||||
|
|
||||||
|
// unify everything to Entity
|
||||||
|
const unifiedContents = driveItems
|
||||||
|
.map((x) => fromDriveItemToEntity(x, this.remoteBaseDir))
|
||||||
|
.filter((x) => x.key !== "/");
|
||||||
|
|
||||||
|
return unifiedContents;
|
||||||
}
|
}
|
||||||
|
|
||||||
// unify everything to Entity
|
async stat(key: string): Promise<Entity> {
|
||||||
const unifiedContents = driveItems
|
await this._init();
|
||||||
.map((x) => fromDriveItemToEntity(x, client.remoteBaseDir))
|
return await this._statFromRoot(getOnedrivePath(key, this.remoteBaseDir));
|
||||||
.filter((x) => x.keyRaw !== "/");
|
}
|
||||||
|
|
||||||
return unifiedContents;
|
async _statFromRoot(key: string): Promise<Entity> {
|
||||||
};
|
// console.info(`remotePath=${remotePath}`);
|
||||||
|
const rsp = await this._getJson(
|
||||||
|
`${key}?$select=cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size`
|
||||||
|
);
|
||||||
|
// console.info(rsp);
|
||||||
|
const driveItem = rsp as DriveItem;
|
||||||
|
const res = fromDriveItemToEntity(driveItem, this.remoteBaseDir);
|
||||||
|
// console.info(res);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
export const getRemoteMeta = async (
|
async mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity> {
|
||||||
client: WrappedOnedriveClient,
|
if (!key.endsWith("/")) {
|
||||||
remotePath: string
|
throw Error(`you should not call mkdir on ${key}`);
|
||||||
) => {
|
}
|
||||||
await client.init();
|
await this._init();
|
||||||
// console.info(`remotePath=${remotePath}`);
|
const uploadFolder = getOnedrivePath(key, this.remoteBaseDir);
|
||||||
const rsp = await client.getJson(
|
console.debug(`mkdir uploadFolder=${uploadFolder}`);
|
||||||
`${remotePath}?$select=cTag,eTag,fileSystemInfo,folder,file,name,parentReference,size`
|
return await this._mkdirFromRoot(uploadFolder, mtime, ctime);
|
||||||
);
|
}
|
||||||
// console.info(rsp);
|
|
||||||
const driveItem = rsp as DriveItem;
|
|
||||||
const res = fromDriveItemToEntity(driveItem, client.remoteBaseDir);
|
|
||||||
// console.info(res);
|
|
||||||
return res;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const uploadToRemote = async (
|
async _mkdirFromRoot(
|
||||||
client: WrappedOnedriveClient,
|
key: string,
|
||||||
fileOrFolderPath: string,
|
mtime?: number,
|
||||||
vault: Vault | undefined,
|
ctime?: number
|
||||||
isRecursively: boolean,
|
): Promise<Entity> {
|
||||||
cipher: Cipher,
|
// console.debug(`foldersCreatedBefore=${Array.from(this.foldersCreatedBefore)}`);
|
||||||
remoteEncryptedKey: string = "",
|
if (this.foldersCreatedBefore.has(key)) {
|
||||||
foldersCreatedBefore: Set<string> | undefined = undefined,
|
// created, pass
|
||||||
uploadRaw: boolean = false,
|
// console.debug(`folder ${key} created.`)
|
||||||
rawContent: string | ArrayBuffer = ""
|
} else {
|
||||||
): Promise<UploadedType> => {
|
// https://stackoverflow.com/questions/56479865/creating-nested-folders-in-one-go-onedrive-api
|
||||||
await client.init();
|
// use PATCH to create folder recursively!!!
|
||||||
|
let playload: any = {
|
||||||
|
folder: {},
|
||||||
|
"@microsoft.graph.conflictBehavior": "replace",
|
||||||
|
};
|
||||||
|
const fileSystemInfo: Record<string, string> = {};
|
||||||
|
if (mtime !== undefined && mtime !== 0) {
|
||||||
|
const mtimeStr = new Date(mtime).toISOString();
|
||||||
|
fileSystemInfo["lastModifiedDateTime"] = mtimeStr;
|
||||||
|
}
|
||||||
|
if (ctime !== undefined && ctime !== 0) {
|
||||||
|
const ctimeStr = new Date(ctime).toISOString();
|
||||||
|
fileSystemInfo["createdDateTime"] = ctimeStr;
|
||||||
|
}
|
||||||
|
if (Object.keys(fileSystemInfo).length > 0) {
|
||||||
|
playload["fileSystemInfo"] = fileSystemInfo;
|
||||||
|
}
|
||||||
|
await this._patchJson(key, playload);
|
||||||
|
}
|
||||||
|
const res = await this._statFromRoot(key);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
let uploadFile = fileOrFolderPath;
|
async writeFile(
|
||||||
if (!cipher.isPasswordEmpty()) {
|
key: string,
|
||||||
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
throw Error(`you should not call writeFile on ${key}`);
|
||||||
|
}
|
||||||
|
await this._init();
|
||||||
|
const uploadFile = getOnedrivePath(key, this.remoteBaseDir);
|
||||||
|
console.debug(`uploadFile=${uploadFile}`);
|
||||||
|
return await this._writeFileFromRoot(
|
||||||
|
uploadFile,
|
||||||
|
content,
|
||||||
|
mtime,
|
||||||
|
ctime,
|
||||||
|
key
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _writeFileFromRoot(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number,
|
||||||
|
origKey: string
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (content.byteLength === 0) {
|
||||||
throw Error(
|
throw Error(
|
||||||
`uploadToRemote(onedrive) you have password but remoteEncryptedKey is empty!`
|
`${origKey}: Empty file is not allowed in OneDrive, and please write something in it.`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
uploadFile = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
uploadFile = getOnedrivePath(uploadFile, client.remoteBaseDir);
|
|
||||||
console.debug(`uploadFile=${uploadFile}`);
|
|
||||||
|
|
||||||
let mtime = 0;
|
const ctimeStr = new Date(ctime).toISOString();
|
||||||
let ctime = 0;
|
const mtimeStr = new Date(mtime).toISOString();
|
||||||
const s = await vault?.adapter?.stat(fileOrFolderPath);
|
|
||||||
if (s !== undefined && s !== null) {
|
|
||||||
mtime = s.mtime;
|
|
||||||
ctime = s.ctime;
|
|
||||||
}
|
|
||||||
const ctimeStr = new Date(ctime).toISOString();
|
|
||||||
const mtimeStr = new Date(mtime).toISOString();
|
|
||||||
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (isFolder && isRecursively) {
|
|
||||||
throw Error("upload function doesn't implement recursive function yet!");
|
|
||||||
} else if (isFolder && !isRecursively) {
|
|
||||||
if (uploadRaw) {
|
|
||||||
throw Error(`you specify uploadRaw, but you also provide a folder key!`);
|
|
||||||
}
|
|
||||||
// folder
|
|
||||||
if (cipher.isPasswordEmpty() || cipher.isFolderAware()) {
|
|
||||||
// if not encrypted, || encrypted isFolderAware, mkdir a remote folder
|
|
||||||
if (foldersCreatedBefore?.has(uploadFile)) {
|
|
||||||
// created, pass
|
|
||||||
} else {
|
|
||||||
// https://stackoverflow.com/questions/56479865/creating-nested-folders-in-one-go-onedrive-api
|
|
||||||
// use PATCH to create folder recursively!!!
|
|
||||||
let k: any = {
|
|
||||||
folder: {},
|
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
|
||||||
};
|
|
||||||
if (mtime !== 0 && ctime !== 0) {
|
|
||||||
k = {
|
|
||||||
folder: {},
|
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
|
||||||
fileSystemInfo: {
|
|
||||||
lastModifiedDateTime: mtimeStr,
|
|
||||||
createdDateTime: ctimeStr,
|
|
||||||
} as FileSystemInfo,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
await client.patchJson(uploadFile, k);
|
|
||||||
}
|
|
||||||
const res = await getRemoteMeta(client, uploadFile);
|
|
||||||
return {
|
|
||||||
entity: res,
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
// if encrypted && !isFolderAware(),
|
|
||||||
// upload a fake, random-size file
|
|
||||||
// with the encrypted file name
|
|
||||||
const byteLengthRandom = getRandomIntInclusive(
|
|
||||||
1,
|
|
||||||
65536 /* max allowed */
|
|
||||||
);
|
|
||||||
const arrBufRandom = await cipher.encryptContent(
|
|
||||||
getRandomArrayBuffer(byteLengthRandom)
|
|
||||||
);
|
|
||||||
|
|
||||||
// an encrypted folder is always small, we just use put here
|
|
||||||
await client.putArrayBuffer(
|
|
||||||
`${uploadFile}:/content?${new URLSearchParams({
|
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
|
||||||
})}`,
|
|
||||||
arrBufRandom
|
|
||||||
);
|
|
||||||
if (mtime !== 0 && ctime !== 0) {
|
|
||||||
await client.patchJson(`${uploadFile}`, {
|
|
||||||
fileSystemInfo: {
|
|
||||||
lastModifiedDateTime: mtimeStr,
|
|
||||||
createdDateTime: ctimeStr,
|
|
||||||
} as FileSystemInfo,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// console.info(uploadResult)
|
|
||||||
const res = await getRemoteMeta(client, uploadFile);
|
|
||||||
return {
|
|
||||||
entity: res,
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// file
|
|
||||||
// we ignore isRecursively parameter here
|
|
||||||
let localContent = undefined;
|
|
||||||
if (uploadRaw) {
|
|
||||||
if (typeof rawContent === "string") {
|
|
||||||
localContent = new TextEncoder().encode(rawContent).buffer;
|
|
||||||
} else {
|
|
||||||
localContent = rawContent;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (vault === undefined) {
|
|
||||||
throw new Error(
|
|
||||||
`the vault variable is not passed but we want to read ${fileOrFolderPath} for OneDrive`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
localContent = await vault.adapter.readBinary(fileOrFolderPath);
|
|
||||||
}
|
|
||||||
let remoteContent = localContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteContent = await cipher.encryptContent(localContent);
|
|
||||||
}
|
|
||||||
|
|
||||||
// no need to create parent folders firstly, cool!
|
// no need to create parent folders firstly, cool!
|
||||||
|
|
||||||
|
@ -857,16 +785,16 @@ export const uploadToRemote = async (
|
||||||
const RANGE_SIZE = MIN_UNIT * 20; // about 6.5536 MB
|
const RANGE_SIZE = MIN_UNIT * 20; // about 6.5536 MB
|
||||||
const DIRECT_UPLOAD_MAX_SIZE = 1000 * 1000 * 4; // 4 Megabyte
|
const DIRECT_UPLOAD_MAX_SIZE = 1000 * 1000 * 4; // 4 Megabyte
|
||||||
|
|
||||||
if (remoteContent.byteLength < DIRECT_UPLOAD_MAX_SIZE) {
|
if (content.byteLength < DIRECT_UPLOAD_MAX_SIZE) {
|
||||||
// directly using put!
|
// directly using put!
|
||||||
await client.putArrayBuffer(
|
await this._putArrayBuffer(
|
||||||
`${uploadFile}:/content?${new URLSearchParams({
|
`${key}:/content?${new URLSearchParams({
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
"@microsoft.graph.conflictBehavior": "replace",
|
||||||
})}`,
|
})}`,
|
||||||
remoteContent
|
content
|
||||||
);
|
);
|
||||||
if (mtime !== 0 && ctime !== 0) {
|
if (mtime !== 0 && ctime !== 0) {
|
||||||
await client.patchJson(`${uploadFile}`, {
|
await this._patchJson(key, {
|
||||||
fileSystemInfo: {
|
fileSystemInfo: {
|
||||||
lastModifiedDateTime: mtimeStr,
|
lastModifiedDateTime: mtimeStr,
|
||||||
createdDateTime: ctimeStr,
|
createdDateTime: ctimeStr,
|
||||||
|
@ -879,13 +807,13 @@ export const uploadToRemote = async (
|
||||||
|
|
||||||
// 1. create uploadSession
|
// 1. create uploadSession
|
||||||
// uploadFile already starts with /drive/special/approot:/${remoteBaseDir}
|
// uploadFile already starts with /drive/special/approot:/${remoteBaseDir}
|
||||||
let k: any = {
|
let playload: any = {
|
||||||
item: {
|
item: {
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
"@microsoft.graph.conflictBehavior": "replace",
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
if (mtime !== 0 && ctime !== 0) {
|
if (mtime !== 0 && ctime !== 0) {
|
||||||
k = {
|
playload = {
|
||||||
item: {
|
item: {
|
||||||
"@microsoft.graph.conflictBehavior": "replace",
|
"@microsoft.graph.conflictBehavior": "replace",
|
||||||
|
|
||||||
|
@ -897,9 +825,9 @@ export const uploadToRemote = async (
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const s: UploadSession = await client.postJson(
|
const s: UploadSession = await this._postJson(
|
||||||
`${uploadFile}:/createUploadSession`,
|
`${key}:/createUploadSession`,
|
||||||
k
|
playload
|
||||||
);
|
);
|
||||||
const uploadUrl = s.uploadUrl!;
|
const uploadUrl = s.uploadUrl!;
|
||||||
console.debug("uploadSession = ");
|
console.debug("uploadSession = ");
|
||||||
|
@ -907,12 +835,12 @@ export const uploadToRemote = async (
|
||||||
|
|
||||||
// 2. upload by ranges
|
// 2. upload by ranges
|
||||||
// convert to uint8
|
// convert to uint8
|
||||||
const uint8 = new Uint8Array(remoteContent);
|
const uint8 = new Uint8Array(content);
|
||||||
|
|
||||||
// upload the ranges one by one
|
// upload the ranges one by one
|
||||||
let rangeStart = 0;
|
let rangeStart = 0;
|
||||||
while (rangeStart < uint8.byteLength) {
|
while (rangeStart < uint8.byteLength) {
|
||||||
await client.putUint8ArrayByRange(
|
await this._putUint8ArrayByRange(
|
||||||
uploadUrl,
|
uploadUrl,
|
||||||
uint8,
|
uint8,
|
||||||
rangeStart,
|
rangeStart,
|
||||||
|
@ -923,132 +851,79 @@ export const uploadToRemote = async (
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = await getRemoteMeta(client, uploadFile);
|
const res = await this._statFromRoot(key);
|
||||||
return {
|
return res;
|
||||||
entity: res,
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const downloadFromRemoteRaw = async (
|
|
||||||
client: WrappedOnedriveClient,
|
|
||||||
remotePath: string
|
|
||||||
): Promise<ArrayBuffer> => {
|
|
||||||
await client.init();
|
|
||||||
const rsp = await client.getJson(
|
|
||||||
`${remotePath}?$select=@microsoft.graph.downloadUrl`
|
|
||||||
);
|
|
||||||
const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
|
|
||||||
if (VALID_REQURL) {
|
|
||||||
const content = (
|
|
||||||
await requestUrl({
|
|
||||||
url: downloadUrl,
|
|
||||||
headers: { "Cache-Control": "no-cache" },
|
|
||||||
})
|
|
||||||
).arrayBuffer;
|
|
||||||
return content;
|
|
||||||
} else {
|
|
||||||
const content = await // cannot set no-cache here, will have cors error
|
|
||||||
(await fetch(downloadUrl)).arrayBuffer();
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const downloadFromRemote = async (
|
|
||||||
client: WrappedOnedriveClient,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault,
|
|
||||||
mtime: number,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
skipSaving: boolean = false
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (!skipSaving) {
|
|
||||||
await mkdirpInVault(fileOrFolderPath, vault);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isFolder) {
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
// mkdirp locally is enough
|
await this._init();
|
||||||
// do nothing here
|
if (key.endsWith("/")) {
|
||||||
return new ArrayBuffer(0);
|
throw new Error(`you should not call readFile on folder ${key}`);
|
||||||
} else {
|
|
||||||
let downloadFile = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
downloadFile = remoteEncryptedKey;
|
|
||||||
}
|
}
|
||||||
downloadFile = getOnedrivePath(downloadFile, client.remoteBaseDir);
|
const downloadFile = getOnedrivePath(key, this.remoteBaseDir);
|
||||||
const remoteContent = await downloadFromRemoteRaw(client, downloadFile);
|
return await this._readFileFromRoot(downloadFile);
|
||||||
let localContent = remoteContent;
|
}
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
localContent = await cipher.decryptContent(remoteContent);
|
async _readFileFromRoot(key: string): Promise<ArrayBuffer> {
|
||||||
|
const rsp = await this._getJson(
|
||||||
|
`${key}?$select=@microsoft.graph.downloadUrl`
|
||||||
|
);
|
||||||
|
const downloadUrl: string = rsp["@microsoft.graph.downloadUrl"];
|
||||||
|
if (VALID_REQURL) {
|
||||||
|
const content = (
|
||||||
|
await requestUrl({
|
||||||
|
url: downloadUrl,
|
||||||
|
headers: { "Cache-Control": "no-cache" },
|
||||||
|
})
|
||||||
|
).arrayBuffer;
|
||||||
|
return content;
|
||||||
|
} else {
|
||||||
|
// cannot set no-cache here, will have cors error
|
||||||
|
const content = await (await fetch(downloadUrl)).arrayBuffer();
|
||||||
|
return content;
|
||||||
}
|
}
|
||||||
if (!skipSaving) {
|
}
|
||||||
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
|
|
||||||
mtime: mtime,
|
async rm(key: string): Promise<void> {
|
||||||
});
|
if (key === "" || key === "/") {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
return localContent;
|
const remoteFileName = getOnedrivePath(key, this.remoteBaseDir);
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const deleteFromRemote = async (
|
await this._init();
|
||||||
client: WrappedOnedriveClient,
|
await this._deleteJson(remoteFileName);
|
||||||
fileOrFolderPath: string,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = ""
|
|
||||||
) => {
|
|
||||||
if (fileOrFolderPath === "/") {
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
let remoteFileName = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteFileName = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
remoteFileName = getOnedrivePath(remoteFileName, client.remoteBaseDir);
|
|
||||||
|
|
||||||
await client.init();
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
await client.deleteJson(remoteFileName);
|
try {
|
||||||
};
|
const k = await this.getUserDisplayName();
|
||||||
|
return k !== "<unknown display name>";
|
||||||
export const checkConnectivity = async (
|
} catch (err) {
|
||||||
client: WrappedOnedriveClient,
|
console.debug(err);
|
||||||
callbackFunc?: any
|
callbackFunc?.(err);
|
||||||
) => {
|
return false;
|
||||||
try {
|
|
||||||
const k = await getUserDisplayName(client);
|
|
||||||
return k !== "<unknown display name>";
|
|
||||||
} catch (err) {
|
|
||||||
console.debug(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
export const getUserDisplayName = async (client: WrappedOnedriveClient) => {
|
async getUserDisplayName() {
|
||||||
await client.init();
|
await this._init();
|
||||||
const res: User = await client.getJson("/me?$select=displayName");
|
const res: User = await this._getJson("/me?$select=displayName");
|
||||||
return res.displayName || "<unknown display name>";
|
return res.displayName || "<unknown display name>";
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
* https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-protocols-oidc#send-a-sign-out-request
|
* https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-protocols-oidc#send-a-sign-out-request
|
||||||
* https://docs.microsoft.com/en-us/graph/api/user-revokesigninsessions
|
* https://docs.microsoft.com/en-us/graph/api/user-revokesigninsessions
|
||||||
* https://docs.microsoft.com/en-us/graph/api/user-invalidateallrefreshtokens
|
* https://docs.microsoft.com/en-us/graph/api/user-invalidateallrefreshtokens
|
||||||
* @param client
|
*/
|
||||||
*/
|
async revokeAuth() {
|
||||||
// export const revokeAuth = async (client: WrappedOnedriveClient) => {
|
// await this._init();
|
||||||
// await client.init();
|
// await this._postJson("/me/revokeSignInSessions", {});
|
||||||
// await client.postJson('/me/revokeSignInSessions', {});
|
throw new Error("Method not implemented.");
|
||||||
// };
|
}
|
||||||
|
|
||||||
export const getRevokeAddr = async () => {
|
async getRevokeAddr() {
|
||||||
return "https://account.live.com/consent/Manage";
|
return "https://account.live.com/consent/Manage";
|
||||||
};
|
}
|
||||||
|
}
|
|
@ -0,0 +1,751 @@
|
||||||
|
import type { _Object, PutObjectCommandInput } from "@aws-sdk/client-s3";
|
||||||
|
import {
|
||||||
|
DeleteObjectCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
HeadObjectCommand,
|
||||||
|
HeadObjectCommandOutput,
|
||||||
|
ListObjectsV2Command,
|
||||||
|
ListObjectsV2CommandInput,
|
||||||
|
PutObjectCommand,
|
||||||
|
S3Client,
|
||||||
|
} from "@aws-sdk/client-s3";
|
||||||
|
import { Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import { HttpRequest, HttpResponse } from "@smithy/protocol-http";
|
||||||
|
import {
|
||||||
|
FetchHttpHandler,
|
||||||
|
FetchHttpHandlerOptions,
|
||||||
|
} from "@smithy/fetch-http-handler";
|
||||||
|
// @ts-ignore
|
||||||
|
import { requestTimeout } from "@smithy/fetch-http-handler/dist-es/request-timeout";
|
||||||
|
import { buildQueryString } from "@smithy/querystring-builder";
|
||||||
|
import { HttpHandlerOptions } from "@aws-sdk/types";
|
||||||
|
import { Buffer } from "buffer";
|
||||||
|
import * as mime from "mime-types";
|
||||||
|
import { Platform, requestUrl, RequestUrlParam } from "obsidian";
|
||||||
|
import { Readable } from "stream";
|
||||||
|
import * as path from "path";
|
||||||
|
import AggregateError from "aggregate-error";
|
||||||
|
import { DEFAULT_CONTENT_TYPE, S3Config, VALID_REQURL } from "./baseTypes";
|
||||||
|
import { bufferToArrayBuffer } from "./misc";
|
||||||
|
import PQueue from "p-queue";
|
||||||
|
|
||||||
|
import { Entity } from "./baseTypes";
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// special handler using Obsidian requestUrl
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is close to origin implementation of FetchHttpHandler
|
||||||
|
* https://github.com/aws/aws-sdk-js-v3/blob/main/packages/fetch-http-handler/src/fetch-http-handler.ts
|
||||||
|
* that is released under Apache 2 License.
|
||||||
|
* But this uses Obsidian requestUrl instead.
|
||||||
|
*/
|
||||||
|
class ObsHttpHandler extends FetchHttpHandler {
|
||||||
|
requestTimeoutInMs: number | undefined;
|
||||||
|
reverseProxyNoSignUrl: string | undefined;
|
||||||
|
constructor(
|
||||||
|
options?: FetchHttpHandlerOptions,
|
||||||
|
reverseProxyNoSignUrl?: string
|
||||||
|
) {
|
||||||
|
super(options);
|
||||||
|
this.requestTimeoutInMs =
|
||||||
|
options === undefined ? undefined : options.requestTimeout;
|
||||||
|
this.reverseProxyNoSignUrl = reverseProxyNoSignUrl;
|
||||||
|
}
|
||||||
|
async handle(
|
||||||
|
request: HttpRequest,
|
||||||
|
{ abortSignal }: HttpHandlerOptions = {}
|
||||||
|
): Promise<{ response: HttpResponse }> {
|
||||||
|
if (abortSignal?.aborted) {
|
||||||
|
const abortError = new Error("Request aborted");
|
||||||
|
abortError.name = "AbortError";
|
||||||
|
return Promise.reject(abortError);
|
||||||
|
}
|
||||||
|
|
||||||
|
let path = request.path;
|
||||||
|
if (request.query) {
|
||||||
|
const queryString = buildQueryString(request.query);
|
||||||
|
if (queryString) {
|
||||||
|
path += `?${queryString}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const { port, method } = request;
|
||||||
|
let url = `${request.protocol}//${request.hostname}${
|
||||||
|
port ? `:${port}` : ""
|
||||||
|
}${path}`;
|
||||||
|
if (
|
||||||
|
this.reverseProxyNoSignUrl !== undefined &&
|
||||||
|
this.reverseProxyNoSignUrl !== ""
|
||||||
|
) {
|
||||||
|
const urlObj = new URL(url);
|
||||||
|
urlObj.host = this.reverseProxyNoSignUrl;
|
||||||
|
url = urlObj.href;
|
||||||
|
}
|
||||||
|
const body =
|
||||||
|
method === "GET" || method === "HEAD" ? undefined : request.body;
|
||||||
|
|
||||||
|
const transformedHeaders: Record<string, string> = {};
|
||||||
|
for (const key of Object.keys(request.headers)) {
|
||||||
|
const keyLower = key.toLowerCase();
|
||||||
|
if (keyLower === "host" || keyLower === "content-length") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
transformedHeaders[keyLower] = request.headers[key];
|
||||||
|
}
|
||||||
|
|
||||||
|
let contentType: string | undefined = undefined;
|
||||||
|
if (transformedHeaders["content-type"] !== undefined) {
|
||||||
|
contentType = transformedHeaders["content-type"];
|
||||||
|
}
|
||||||
|
|
||||||
|
let transformedBody: any = body;
|
||||||
|
if (ArrayBuffer.isView(body)) {
|
||||||
|
transformedBody = bufferToArrayBuffer(body);
|
||||||
|
}
|
||||||
|
|
||||||
|
const param: RequestUrlParam = {
|
||||||
|
body: transformedBody,
|
||||||
|
headers: transformedHeaders,
|
||||||
|
method: method,
|
||||||
|
url: url,
|
||||||
|
contentType: contentType,
|
||||||
|
};
|
||||||
|
|
||||||
|
const raceOfPromises = [
|
||||||
|
requestUrl(param).then((rsp) => {
|
||||||
|
const headers = rsp.headers;
|
||||||
|
const headersLower: Record<string, string> = {};
|
||||||
|
for (const key of Object.keys(headers)) {
|
||||||
|
headersLower[key.toLowerCase()] = headers[key];
|
||||||
|
}
|
||||||
|
const stream = new ReadableStream<Uint8Array>({
|
||||||
|
start(controller) {
|
||||||
|
controller.enqueue(new Uint8Array(rsp.arrayBuffer));
|
||||||
|
controller.close();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
response: new HttpResponse({
|
||||||
|
headers: headersLower,
|
||||||
|
statusCode: rsp.status,
|
||||||
|
body: stream,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
requestTimeout(this.requestTimeoutInMs),
|
||||||
|
];
|
||||||
|
|
||||||
|
if (abortSignal) {
|
||||||
|
raceOfPromises.push(
|
||||||
|
new Promise<never>((resolve, reject) => {
|
||||||
|
abortSignal.onabort = () => {
|
||||||
|
const abortError = new Error("Request aborted");
|
||||||
|
abortError.name = "AbortError";
|
||||||
|
reject(abortError);
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Promise.race(raceOfPromises);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// other stuffs
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
export const simpleTransRemotePrefix = (x: string) => {
|
||||||
|
if (x === undefined) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
let y = path.posix.normalize(x.trim());
|
||||||
|
if (y === undefined || y === "" || y === "/" || y === ".") {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
if (y.startsWith("/")) {
|
||||||
|
y = y.slice(1);
|
||||||
|
}
|
||||||
|
if (!y.endsWith("/")) {
|
||||||
|
y = `${y}/`;
|
||||||
|
}
|
||||||
|
return y;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const DEFAULT_S3_CONFIG: S3Config = {
|
||||||
|
s3Endpoint: "",
|
||||||
|
s3Region: "",
|
||||||
|
s3AccessKeyID: "",
|
||||||
|
s3SecretAccessKey: "",
|
||||||
|
s3BucketName: "",
|
||||||
|
bypassCorsLocally: true,
|
||||||
|
partsConcurrency: 20,
|
||||||
|
forcePathStyle: false,
|
||||||
|
remotePrefix: "",
|
||||||
|
useAccurateMTime: false, // it causes money, disable by default
|
||||||
|
reverseProxyNoSignUrl: "",
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Body of resp of aws GetObject has mix types
|
||||||
|
* and we want to get ArrayBuffer here.
|
||||||
|
* See https://github.com/aws/aws-sdk-js-v3/issues/1877
|
||||||
|
* @param b The Body of GetObject
|
||||||
|
* @returns Promise<ArrayBuffer>
|
||||||
|
*/
|
||||||
|
const getObjectBodyToArrayBuffer = async (
|
||||||
|
b: Readable | ReadableStream | Blob | undefined
|
||||||
|
) => {
|
||||||
|
if (b === undefined) {
|
||||||
|
throw Error(`ObjectBody is undefined and don't know how to deal with it`);
|
||||||
|
}
|
||||||
|
if (b instanceof Readable) {
|
||||||
|
return (await new Promise((resolve, reject) => {
|
||||||
|
const chunks: Uint8Array[] = [];
|
||||||
|
b.on("data", (chunk) => chunks.push(chunk));
|
||||||
|
b.on("error", reject);
|
||||||
|
b.on("end", () => resolve(bufferToArrayBuffer(Buffer.concat(chunks))));
|
||||||
|
})) as ArrayBuffer;
|
||||||
|
} else if (b instanceof ReadableStream) {
|
||||||
|
return await new Response(b, {}).arrayBuffer();
|
||||||
|
} else if (b instanceof Blob) {
|
||||||
|
return await b.arrayBuffer();
|
||||||
|
} else {
|
||||||
|
throw TypeError(`The type of ${b} is not one of the supported types`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const getS3Client = (s3Config: S3Config) => {
|
||||||
|
let endpoint = s3Config.s3Endpoint;
|
||||||
|
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
|
||||||
|
endpoint = `https://${endpoint}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
let s3Client: S3Client;
|
||||||
|
if (VALID_REQURL && s3Config.bypassCorsLocally) {
|
||||||
|
s3Client = new S3Client({
|
||||||
|
region: s3Config.s3Region,
|
||||||
|
endpoint: endpoint,
|
||||||
|
forcePathStyle: s3Config.forcePathStyle,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: s3Config.s3AccessKeyID,
|
||||||
|
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||||
|
},
|
||||||
|
requestHandler: new ObsHttpHandler(
|
||||||
|
undefined,
|
||||||
|
s3Config.reverseProxyNoSignUrl
|
||||||
|
),
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
s3Client = new S3Client({
|
||||||
|
region: s3Config.s3Region,
|
||||||
|
endpoint: endpoint,
|
||||||
|
forcePathStyle: s3Config.forcePathStyle,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: s3Config.s3AccessKeyID,
|
||||||
|
secretAccessKey: s3Config.s3SecretAccessKey,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
s3Client.middlewareStack.add(
|
||||||
|
(next, context) => (args) => {
|
||||||
|
(args.request as any).headers["cache-control"] = "no-cache";
|
||||||
|
return next(args);
|
||||||
|
},
|
||||||
|
{
|
||||||
|
step: "build",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
return s3Client;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getLocalNoPrefixPath = (
|
||||||
|
fileOrFolderPathWithRemotePrefix: string,
|
||||||
|
remotePrefix: string
|
||||||
|
) => {
|
||||||
|
if (
|
||||||
|
!(
|
||||||
|
fileOrFolderPathWithRemotePrefix === `${remotePrefix}` ||
|
||||||
|
fileOrFolderPathWithRemotePrefix.startsWith(`${remotePrefix}`)
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw Error(
|
||||||
|
`"${fileOrFolderPathWithRemotePrefix}" doesn't starts with "${remotePrefix}"`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length);
|
||||||
|
};
|
||||||
|
|
||||||
|
const getRemoteWithPrefixPath = (
|
||||||
|
fileOrFolderPath: string,
|
||||||
|
remotePrefix: string
|
||||||
|
) => {
|
||||||
|
if (remotePrefix === undefined || remotePrefix === "") {
|
||||||
|
return fileOrFolderPath;
|
||||||
|
}
|
||||||
|
let key = fileOrFolderPath;
|
||||||
|
if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
|
||||||
|
// special
|
||||||
|
key = remotePrefix;
|
||||||
|
}
|
||||||
|
if (!fileOrFolderPath.startsWith("/")) {
|
||||||
|
key = `${remotePrefix}${fileOrFolderPath}`;
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
};
|
||||||
|
|
||||||
|
const fromS3ObjectToEntity = (
|
||||||
|
x: _Object,
|
||||||
|
remotePrefix: string,
|
||||||
|
mtimeRecords: Record<string, number>,
|
||||||
|
ctimeRecords: Record<string, number>
|
||||||
|
) => {
|
||||||
|
// console.debug(`fromS3ObjectToEntity: ${x.Key!}, ${JSON.stringify(x,null,2)}`);
|
||||||
|
// S3 officially only supports seconds precision!!!!!
|
||||||
|
const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000;
|
||||||
|
let mtimeCli = mtimeSvr;
|
||||||
|
if (x.Key! in mtimeRecords) {
|
||||||
|
const m2 = mtimeRecords[x.Key!];
|
||||||
|
if (m2 !== 0) {
|
||||||
|
// to be compatible with RClone, we read and store the time in seconds in new version!
|
||||||
|
if (m2 >= 1000000000000) {
|
||||||
|
// it's a millsecond, uploaded by old codes..
|
||||||
|
mtimeCli = m2;
|
||||||
|
} else {
|
||||||
|
// it's a second, uploaded by new codes of the plugin from March 24, 2024
|
||||||
|
mtimeCli = m2 * 1000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const key = getLocalNoPrefixPath(x.Key!, remotePrefix); // we remove prefix here
|
||||||
|
const r: Entity = {
|
||||||
|
key: key, // from s3's repsective, the keyRaw is the key, we will change it in decyption
|
||||||
|
keyRaw: key,
|
||||||
|
mtimeSvr: mtimeSvr,
|
||||||
|
mtimeCli: mtimeCli,
|
||||||
|
sizeRaw: x.Size!,
|
||||||
|
size: x.Size!, // from s3's repsective, the sizeRaw is the size, we will change it in decyption
|
||||||
|
etag: x.ETag,
|
||||||
|
synthesizedFolder: false,
|
||||||
|
};
|
||||||
|
return r;
|
||||||
|
};
|
||||||
|
|
||||||
|
const fromS3HeadObjectToEntity = (
|
||||||
|
fileOrFolderPathWithRemotePrefix: string,
|
||||||
|
x: HeadObjectCommandOutput,
|
||||||
|
remotePrefix: string
|
||||||
|
) => {
|
||||||
|
// console.debug(`fromS3HeadObjectToEntity: ${fileOrFolderPathWithRemotePrefix}: ${JSON.stringify(x,null,2)}`);
|
||||||
|
// S3 officially only supports seconds precision!!!!!
|
||||||
|
const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000;
|
||||||
|
let mtimeCli = mtimeSvr;
|
||||||
|
if (x.Metadata !== undefined) {
|
||||||
|
const m2 = Math.floor(
|
||||||
|
parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0")
|
||||||
|
);
|
||||||
|
if (m2 !== 0) {
|
||||||
|
// to be compatible with RClone, we read and store the time in seconds in new version!
|
||||||
|
if (m2 >= 1000000000000) {
|
||||||
|
// it's a millsecond, uploaded by old codes..
|
||||||
|
mtimeCli = m2;
|
||||||
|
} else {
|
||||||
|
// it's a second, uploaded by new codes of the plugin from March 24, 2024
|
||||||
|
mtimeCli = m2 * 1000;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// console.debug(
|
||||||
|
// `fromS3HeadObjectToEntity, fileOrFolderPathWithRemotePrefix=${fileOrFolderPathWithRemotePrefix}, remotePrefix=${remotePrefix}, x=${JSON.stringify(
|
||||||
|
// x
|
||||||
|
// )} `
|
||||||
|
// );
|
||||||
|
const key = getLocalNoPrefixPath(
|
||||||
|
fileOrFolderPathWithRemotePrefix,
|
||||||
|
remotePrefix
|
||||||
|
);
|
||||||
|
// console.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`);
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
keyRaw: key,
|
||||||
|
mtimeSvr: mtimeSvr,
|
||||||
|
mtimeCli: mtimeCli,
|
||||||
|
sizeRaw: x.ContentLength,
|
||||||
|
size: x.ContentLength,
|
||||||
|
etag: x.ETag,
|
||||||
|
synthesizedFolder: false,
|
||||||
|
} as Entity;
|
||||||
|
};
|
||||||
|
|
||||||
|
export class FakeFsS3 extends FakeFs {
|
||||||
|
s3Config: S3Config;
|
||||||
|
s3Client: S3Client;
|
||||||
|
kind: "s3";
|
||||||
|
constructor(s3Config: S3Config) {
|
||||||
|
super();
|
||||||
|
this.s3Config = s3Config;
|
||||||
|
this.s3Client = getS3Client(s3Config);
|
||||||
|
this.kind = "s3";
|
||||||
|
}
|
||||||
|
|
||||||
|
async walk(): Promise<Entity[]> {
|
||||||
|
const res = (await this._walkFromRoot(this.s3Config.remotePrefix)).filter(
|
||||||
|
(x) => x.key !== "" && x.key !== "/"
|
||||||
|
);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the input key contains basedir (prefix),
|
||||||
|
* but the result doesn't contain it.
|
||||||
|
*/
|
||||||
|
async _walkFromRoot(prefixOfRawKeys: string | undefined) {
|
||||||
|
const confCmd = {
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
} as ListObjectsV2CommandInput;
|
||||||
|
if (prefixOfRawKeys !== undefined && prefixOfRawKeys !== "") {
|
||||||
|
confCmd.Prefix = prefixOfRawKeys;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contents = [] as _Object[];
|
||||||
|
const mtimeRecords: Record<string, number> = {};
|
||||||
|
const ctimeRecords: Record<string, number> = {};
|
||||||
|
const queueHead = new PQueue({
|
||||||
|
concurrency: this.s3Config.partsConcurrency,
|
||||||
|
autoStart: true,
|
||||||
|
});
|
||||||
|
queueHead.on("error", (error) => {
|
||||||
|
queueHead.pause();
|
||||||
|
queueHead.clear();
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
|
||||||
|
let isTruncated = true;
|
||||||
|
do {
|
||||||
|
const rsp = await this.s3Client.send(new ListObjectsV2Command(confCmd));
|
||||||
|
|
||||||
|
if (rsp.$metadata.httpStatusCode !== 200) {
|
||||||
|
throw Error("some thing bad while listing remote!");
|
||||||
|
}
|
||||||
|
if (rsp.Contents === undefined) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
contents.push(...rsp.Contents);
|
||||||
|
|
||||||
|
if (this.s3Config.useAccurateMTime) {
|
||||||
|
// head requests of all objects, love it
|
||||||
|
for (const content of rsp.Contents) {
|
||||||
|
queueHead.add(async () => {
|
||||||
|
const rspHead = await this.s3Client.send(
|
||||||
|
new HeadObjectCommand({
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
Key: content.Key,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
if (rspHead.$metadata.httpStatusCode !== 200) {
|
||||||
|
throw Error("some thing bad while heading single object!");
|
||||||
|
}
|
||||||
|
if (rspHead.Metadata === undefined) {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
mtimeRecords[content.Key!] = Math.floor(
|
||||||
|
parseFloat(
|
||||||
|
rspHead.Metadata.mtime || rspHead.Metadata.MTime || "0"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
ctimeRecords[content.Key!] = Math.floor(
|
||||||
|
parseFloat(
|
||||||
|
rspHead.Metadata.ctime || rspHead.Metadata.CTime || "0"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isTruncated = rsp.IsTruncated ?? false;
|
||||||
|
confCmd.ContinuationToken = rsp.NextContinuationToken;
|
||||||
|
if (
|
||||||
|
isTruncated &&
|
||||||
|
(confCmd.ContinuationToken === undefined ||
|
||||||
|
confCmd.ContinuationToken === "")
|
||||||
|
) {
|
||||||
|
throw Error("isTruncated is true but no continuationToken provided");
|
||||||
|
}
|
||||||
|
} while (isTruncated);
|
||||||
|
|
||||||
|
// wait for any head requests
|
||||||
|
await queueHead.onIdle();
|
||||||
|
|
||||||
|
// ensemble fake rsp
|
||||||
|
// in the end, we need to transform the response list
|
||||||
|
// back to the local contents-alike list
|
||||||
|
return contents.map((x) =>
|
||||||
|
fromS3ObjectToEntity(
|
||||||
|
x,
|
||||||
|
this.s3Config.remotePrefix ?? "",
|
||||||
|
mtimeRecords,
|
||||||
|
ctimeRecords
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(key: string): Promise<Entity> {
|
||||||
|
let keyFullPath = key;
|
||||||
|
keyFullPath = getRemoteWithPrefixPath(
|
||||||
|
keyFullPath,
|
||||||
|
this.s3Config.remotePrefix ?? ""
|
||||||
|
);
|
||||||
|
return await this._statFromRoot(keyFullPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the input key contains basedir (prefix),
|
||||||
|
* but the result doesn't contain it.
|
||||||
|
*/
|
||||||
|
async _statFromRoot(key: string): Promise<Entity> {
|
||||||
|
if (
|
||||||
|
this.s3Config.remotePrefix !== undefined &&
|
||||||
|
this.s3Config.remotePrefix !== "" &&
|
||||||
|
!key.startsWith(this.s3Config.remotePrefix)
|
||||||
|
) {
|
||||||
|
throw Error(`_statFromRoot should only accept prefix-ed path`);
|
||||||
|
}
|
||||||
|
const res = await this.s3Client.send(
|
||||||
|
new HeadObjectCommand({
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
Key: key,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
return fromS3HeadObjectToEntity(key, res, this.s3Config.remotePrefix ?? "");
|
||||||
|
}
|
||||||
|
|
||||||
|
async mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity> {
|
||||||
|
if (!key.endsWith("/")) {
|
||||||
|
throw new Error(`You should not call mkdir on ${key}!`);
|
||||||
|
}
|
||||||
|
const uploadFile = getRemoteWithPrefixPath(
|
||||||
|
key,
|
||||||
|
this.s3Config.remotePrefix ?? ""
|
||||||
|
);
|
||||||
|
return await this._mkdirFromRoot(uploadFile, mtime, ctime);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _mkdirFromRoot(key: string, mtime?: number, ctime?: number) {
|
||||||
|
if (
|
||||||
|
this.s3Config.remotePrefix !== undefined &&
|
||||||
|
this.s3Config.remotePrefix !== "" &&
|
||||||
|
!key.startsWith(this.s3Config.remotePrefix)
|
||||||
|
) {
|
||||||
|
throw Error(`_mkdirFromRoot should only accept prefix-ed path`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = DEFAULT_CONTENT_TYPE;
|
||||||
|
const p: PutObjectCommandInput = {
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
Key: key,
|
||||||
|
Body: "",
|
||||||
|
ContentType: contentType,
|
||||||
|
ContentLength: 0, // interesting we need to set this to avoid the warning
|
||||||
|
};
|
||||||
|
const metadata: Record<string, string> = {};
|
||||||
|
if (mtime !== undefined && mtime !== 0) {
|
||||||
|
metadata["MTime"] = `${mtime / 1000.0}`;
|
||||||
|
}
|
||||||
|
if (ctime !== undefined && ctime !== 0) {
|
||||||
|
metadata["CTime"] = `${ctime / 1000.0}`;
|
||||||
|
}
|
||||||
|
if (Object.keys(metadata).length > 0) {
|
||||||
|
p["Metadata"] = metadata;
|
||||||
|
}
|
||||||
|
await this.s3Client.send(new PutObjectCommand(p));
|
||||||
|
return await this._statFromRoot(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeFile(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
const uploadFile = getRemoteWithPrefixPath(
|
||||||
|
key,
|
||||||
|
this.s3Config.remotePrefix ?? ""
|
||||||
|
);
|
||||||
|
const res = await this._writeFileFromRoot(
|
||||||
|
uploadFile,
|
||||||
|
content,
|
||||||
|
mtime,
|
||||||
|
ctime
|
||||||
|
);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the input key contains basedir (prefix),
|
||||||
|
* but the result doesn't contain it.
|
||||||
|
*/
|
||||||
|
async _writeFileFromRoot(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (
|
||||||
|
this.s3Config.remotePrefix !== undefined &&
|
||||||
|
this.s3Config.remotePrefix !== "" &&
|
||||||
|
!key.startsWith(this.s3Config.remotePrefix)
|
||||||
|
) {
|
||||||
|
throw Error(`_writeFileFromRoot should only accept prefix-ed path`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const bytesIn5MB = 5242880;
|
||||||
|
const body = new Uint8Array(content);
|
||||||
|
|
||||||
|
let contentType = DEFAULT_CONTENT_TYPE;
|
||||||
|
contentType =
|
||||||
|
mime.contentType(mime.lookup(key) || DEFAULT_CONTENT_TYPE) ||
|
||||||
|
DEFAULT_CONTENT_TYPE;
|
||||||
|
|
||||||
|
const upload = new Upload({
|
||||||
|
client: this.s3Client,
|
||||||
|
queueSize: this.s3Config.partsConcurrency, // concurrency
|
||||||
|
partSize: bytesIn5MB, // minimal 5MB by default
|
||||||
|
leavePartsOnError: false,
|
||||||
|
params: {
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
Key: key,
|
||||||
|
Body: body,
|
||||||
|
ContentType: contentType,
|
||||||
|
Metadata: {
|
||||||
|
MTime: `${mtime / 1000.0}`,
|
||||||
|
CTime: `${ctime / 1000.0}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
upload.on("httpUploadProgress", (progress) => {
|
||||||
|
// console.info(progress);
|
||||||
|
});
|
||||||
|
await upload.done();
|
||||||
|
|
||||||
|
return await this._statFromRoot(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
throw new Error(`you should not call readFile on folder ${key}`);
|
||||||
|
}
|
||||||
|
const downloadFile = getRemoteWithPrefixPath(
|
||||||
|
key,
|
||||||
|
this.s3Config.remotePrefix ?? ""
|
||||||
|
);
|
||||||
|
|
||||||
|
return await this._readFileFromRoot(downloadFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _readFileFromRoot(key: string): Promise<ArrayBuffer> {
|
||||||
|
if (
|
||||||
|
this.s3Config.remotePrefix !== undefined &&
|
||||||
|
this.s3Config.remotePrefix !== "" &&
|
||||||
|
!key.startsWith(this.s3Config.remotePrefix)
|
||||||
|
) {
|
||||||
|
throw Error(`_readFileFromRoot should only accept prefix-ed path`);
|
||||||
|
}
|
||||||
|
const data = await this.s3Client.send(
|
||||||
|
new GetObjectCommand({
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
Key: key,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
|
||||||
|
return bodyContents;
|
||||||
|
}
|
||||||
|
|
||||||
|
async rm(key: string): Promise<void> {
|
||||||
|
if (key === "/") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const remoteFileName = getRemoteWithPrefixPath(
|
||||||
|
key,
|
||||||
|
this.s3Config.remotePrefix ?? ""
|
||||||
|
);
|
||||||
|
|
||||||
|
await this.s3Client.send(
|
||||||
|
new DeleteObjectCommand({
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
Key: remoteFileName,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// TODO: do we need to delete folder recursively?
|
||||||
|
// maybe we should not
|
||||||
|
// because the outer sync algorithm should do that
|
||||||
|
// (await this._walkFromRoot(remoteFileName)).map(...)
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
// TODO: no universal way now, just check this in connectivity
|
||||||
|
if (Platform.isIosApp && this.s3Config.s3Endpoint.startsWith("http://")) {
|
||||||
|
throw Error(
|
||||||
|
`Your s3 endpoint could only be https, not http, because of the iOS restriction.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// const results = await this.s3Client.send(
|
||||||
|
// new HeadBucketCommand({ Bucket: this.s3Config.s3BucketName })
|
||||||
|
// );
|
||||||
|
// very simplified version of listing objects
|
||||||
|
const confCmd = {
|
||||||
|
Bucket: this.s3Config.s3BucketName,
|
||||||
|
} as ListObjectsV2CommandInput;
|
||||||
|
const results = await this.s3Client.send(
|
||||||
|
new ListObjectsV2Command(confCmd)
|
||||||
|
);
|
||||||
|
|
||||||
|
if (
|
||||||
|
results === undefined ||
|
||||||
|
results.$metadata === undefined ||
|
||||||
|
results.$metadata.httpStatusCode === undefined
|
||||||
|
) {
|
||||||
|
const err = "results or $metadata or httStatusCode is undefined";
|
||||||
|
console.debug(err);
|
||||||
|
if (callbackFunc !== undefined) {
|
||||||
|
callbackFunc(err);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return results.$metadata.httpStatusCode === 200;
|
||||||
|
} catch (err: any) {
|
||||||
|
console.debug(err);
|
||||||
|
if (callbackFunc !== undefined) {
|
||||||
|
if (this.s3Config.s3Endpoint.contains(this.s3Config.s3BucketName)) {
|
||||||
|
const err2 = new AggregateError([
|
||||||
|
err,
|
||||||
|
new Error(
|
||||||
|
"Maybe you've included the bucket name inside the endpoint setting. Please remove the bucket name and try again."
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
callbackFunc(err2);
|
||||||
|
} else {
|
||||||
|
callbackFunc(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserDisplayName(): Promise<string> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async revokeAuth() {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,494 @@
|
||||||
|
import { getReasonPhrase } from "http-status-codes/build/cjs/utils-functions";
|
||||||
|
import { Buffer } from "buffer";
|
||||||
|
import cloneDeep from "lodash/cloneDeep";
|
||||||
|
import { Queue } from "@fyears/tsqueue";
|
||||||
|
import chunk from "lodash/chunk";
|
||||||
|
import flatten from "lodash/flatten";
|
||||||
|
import { Platform, requestUrl } from "obsidian";
|
||||||
|
import { FakeFs } from "./fsAll";
|
||||||
|
import { bufferToArrayBuffer } from "./misc";
|
||||||
|
import { Entity, VALID_REQURL, WebdavConfig } from "./baseTypes";
|
||||||
|
import type {
|
||||||
|
FileStat,
|
||||||
|
WebDAVClient,
|
||||||
|
RequestOptionsWithState,
|
||||||
|
// Response,
|
||||||
|
// ResponseDataDetailed,
|
||||||
|
} from "webdav";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* https://stackoverflow.com/questions/32850898/how-to-check-if-a-string-has-any-non-iso-8859-1-characters-with-javascript
|
||||||
|
* @param str
|
||||||
|
* @returns true if all are iso 8859 1 chars
|
||||||
|
*/
|
||||||
|
function onlyAscii(str: string) {
|
||||||
|
return !/[^\u0000-\u00ff]/g.test(str);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* https://stackoverflow.com/questions/12539574/
|
||||||
|
* @param obj
|
||||||
|
* @returns
|
||||||
|
*/
|
||||||
|
function objKeyToLower(obj: Record<string, string>) {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(obj).map(([k, v]) => [k.toLowerCase(), v])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// @ts-ignore
|
||||||
|
import { getPatcher } from "webdav/dist/web/index.js";
|
||||||
|
if (VALID_REQURL) {
|
||||||
|
getPatcher().patch(
|
||||||
|
"request",
|
||||||
|
async (options: RequestOptionsWithState): Promise<Response> => {
|
||||||
|
const transformedHeaders = objKeyToLower({ ...options.headers });
|
||||||
|
delete transformedHeaders["host"];
|
||||||
|
delete transformedHeaders["content-length"];
|
||||||
|
|
||||||
|
const reqContentType =
|
||||||
|
transformedHeaders["accept"] ?? transformedHeaders["content-type"];
|
||||||
|
|
||||||
|
const retractedHeaders = { ...transformedHeaders };
|
||||||
|
if (retractedHeaders.hasOwnProperty("authorization")) {
|
||||||
|
retractedHeaders["authorization"] = "<retracted>";
|
||||||
|
}
|
||||||
|
|
||||||
|
console.debug(`before request:`);
|
||||||
|
console.debug(`url: ${options.url}`);
|
||||||
|
console.debug(`method: ${options.method}`);
|
||||||
|
console.debug(`headers: ${JSON.stringify(retractedHeaders, null, 2)}`);
|
||||||
|
console.debug(`reqContentType: ${reqContentType}`);
|
||||||
|
|
||||||
|
let r = await requestUrl({
|
||||||
|
url: options.url,
|
||||||
|
method: options.method,
|
||||||
|
body: options.data as string | ArrayBuffer,
|
||||||
|
headers: transformedHeaders,
|
||||||
|
contentType: reqContentType,
|
||||||
|
throw: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (
|
||||||
|
r.status === 401 &&
|
||||||
|
Platform.isIosApp &&
|
||||||
|
!options.url.endsWith("/") &&
|
||||||
|
!options.url.endsWith(".md") &&
|
||||||
|
options.method.toUpperCase() === "PROPFIND"
|
||||||
|
) {
|
||||||
|
// don't ask me why,
|
||||||
|
// some webdav servers have some mysterious behaviours,
|
||||||
|
// if a folder doesn't exist without slash, the servers return 401 instead of 404
|
||||||
|
// here is a dirty hack that works
|
||||||
|
console.debug(`so we have 401, try appending request url with slash`);
|
||||||
|
r = await requestUrl({
|
||||||
|
url: `${options.url}/`,
|
||||||
|
method: options.method,
|
||||||
|
body: options.data as string | ArrayBuffer,
|
||||||
|
headers: transformedHeaders,
|
||||||
|
contentType: reqContentType,
|
||||||
|
throw: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.debug(`after request:`);
|
||||||
|
const rspHeaders = objKeyToLower({ ...r.headers });
|
||||||
|
console.debug(`rspHeaders: ${JSON.stringify(rspHeaders, null, 2)}`);
|
||||||
|
for (let key in rspHeaders) {
|
||||||
|
if (rspHeaders.hasOwnProperty(key)) {
|
||||||
|
// avoid the error:
|
||||||
|
// Failed to read the 'headers' property from 'ResponseInit': String contains non ISO-8859-1 code point.
|
||||||
|
// const possibleNonAscii = [
|
||||||
|
// "Content-Disposition",
|
||||||
|
// "X-Accel-Redirect",
|
||||||
|
// "X-Outfilename",
|
||||||
|
// "X-Sendfile"
|
||||||
|
// ];
|
||||||
|
// for (const p of possibleNonAscii) {
|
||||||
|
// if (key === p || key === p.toLowerCase()) {
|
||||||
|
// rspHeaders[key] = encodeURIComponent(rspHeaders[key]);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
if (!onlyAscii(rspHeaders[key])) {
|
||||||
|
console.debug(`rspHeaders[key] needs encode: ${key}`);
|
||||||
|
rspHeaders[key] = encodeURIComponent(rspHeaders[key]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let r2: Response | undefined = undefined;
|
||||||
|
const statusText = getReasonPhrase(r.status);
|
||||||
|
console.debug(`statusText: ${statusText}`);
|
||||||
|
if ([101, 103, 204, 205, 304].includes(r.status)) {
|
||||||
|
// A null body status is a status that is 101, 103, 204, 205, or 304.
|
||||||
|
// https://fetch.spec.whatwg.org/#statuses
|
||||||
|
// fix this: Failed to construct 'Response': Response with null body status cannot have body
|
||||||
|
r2 = new Response(null, {
|
||||||
|
status: r.status,
|
||||||
|
statusText: statusText,
|
||||||
|
headers: rspHeaders,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
r2 = new Response(r.arrayBuffer, {
|
||||||
|
status: r.status,
|
||||||
|
statusText: statusText,
|
||||||
|
headers: rspHeaders,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return r2;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// @ts-ignore
|
||||||
|
import { AuthType, BufferLike, createClient } from "webdav/dist/web/index.js";
|
||||||
|
|
||||||
|
export const DEFAULT_WEBDAV_CONFIG = {
|
||||||
|
address: "",
|
||||||
|
username: "",
|
||||||
|
password: "",
|
||||||
|
authType: "basic",
|
||||||
|
manualRecursive: true,
|
||||||
|
depth: "manual_1",
|
||||||
|
remoteBaseDir: "",
|
||||||
|
} as WebdavConfig;
|
||||||
|
|
||||||
|
const getWebdavPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
||||||
|
let key = fileOrFolderPath;
|
||||||
|
if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
|
||||||
|
// special
|
||||||
|
key = `/${remoteBaseDir}/`;
|
||||||
|
} else if (fileOrFolderPath.startsWith("/")) {
|
||||||
|
console.warn(
|
||||||
|
`why the path ${fileOrFolderPath} starts with '/'? but we just go on.`
|
||||||
|
);
|
||||||
|
key = `/${remoteBaseDir}${fileOrFolderPath}`;
|
||||||
|
} else {
|
||||||
|
key = `/${remoteBaseDir}/${fileOrFolderPath}`;
|
||||||
|
}
|
||||||
|
return key;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
||||||
|
if (
|
||||||
|
!(
|
||||||
|
fileOrFolderPath === `/${remoteBaseDir}` ||
|
||||||
|
fileOrFolderPath.startsWith(`/${remoteBaseDir}/`)
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw Error(
|
||||||
|
`"${fileOrFolderPath}" doesn't starts with "/${remoteBaseDir}/"`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length);
|
||||||
|
};
|
||||||
|
|
||||||
|
const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string): Entity => {
|
||||||
|
let key = getNormPath(x.filename, remoteBaseDir);
|
||||||
|
if (x.type === "directory" && !key.endsWith("/")) {
|
||||||
|
key = `${key}/`;
|
||||||
|
}
|
||||||
|
const mtimeSvr = Date.parse(x.lastmod).valueOf();
|
||||||
|
return {
|
||||||
|
key: key,
|
||||||
|
keyRaw: key,
|
||||||
|
mtimeSvr: mtimeSvr,
|
||||||
|
mtimeCli: mtimeSvr, // TODO: no universal way to set mtime in webdav
|
||||||
|
size: x.size,
|
||||||
|
sizeRaw: x.size,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export class FakeFsWebdav extends FakeFs {
|
||||||
|
kind: "webdav";
|
||||||
|
|
||||||
|
webdavConfig: WebdavConfig;
|
||||||
|
remoteBaseDir: string;
|
||||||
|
client!: WebDAVClient;
|
||||||
|
vaultFolderExists: boolean;
|
||||||
|
saveUpdatedConfigFunc: () => Promise<any>;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
webdavConfig: WebdavConfig,
|
||||||
|
vaultName: string,
|
||||||
|
saveUpdatedConfigFunc: () => Promise<any>
|
||||||
|
) {
|
||||||
|
super();
|
||||||
|
this.kind = "webdav";
|
||||||
|
this.webdavConfig = cloneDeep(webdavConfig);
|
||||||
|
this.webdavConfig.address = encodeURI(this.webdavConfig.address);
|
||||||
|
this.remoteBaseDir = this.webdavConfig.remoteBaseDir || vaultName || "";
|
||||||
|
this.vaultFolderExists = false;
|
||||||
|
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
||||||
|
}
|
||||||
|
|
||||||
|
async _init() {
|
||||||
|
// init client if not inited
|
||||||
|
if (this.client !== undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Platform.isIosApp && !this.webdavConfig.address.startsWith("https")) {
|
||||||
|
throw Error(
|
||||||
|
`Your webdav address could only be https, not http, because of the iOS restriction.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers = {
|
||||||
|
"Cache-Control": "no-cache",
|
||||||
|
};
|
||||||
|
if (
|
||||||
|
this.webdavConfig.username !== "" &&
|
||||||
|
this.webdavConfig.password !== ""
|
||||||
|
) {
|
||||||
|
this.client = createClient(this.webdavConfig.address, {
|
||||||
|
username: this.webdavConfig.username,
|
||||||
|
password: this.webdavConfig.password,
|
||||||
|
headers: headers,
|
||||||
|
authType:
|
||||||
|
this.webdavConfig.authType === "digest"
|
||||||
|
? AuthType.Digest
|
||||||
|
: AuthType.Password,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
console.info("no password");
|
||||||
|
this.client = createClient(this.webdavConfig.address, {
|
||||||
|
headers: headers,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// check vault folder
|
||||||
|
if (this.vaultFolderExists) {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
const res = await this.client.exists(`/${this.remoteBaseDir}/`);
|
||||||
|
if (res) {
|
||||||
|
// console.info("remote vault folder exits!");
|
||||||
|
this.vaultFolderExists = true;
|
||||||
|
} else {
|
||||||
|
console.info("remote vault folder not exists, creating");
|
||||||
|
await this.client.createDirectory(`/${this.remoteBaseDir}/`);
|
||||||
|
console.info("remote vault folder created!");
|
||||||
|
this.vaultFolderExists = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjust depth parameter
|
||||||
|
if (
|
||||||
|
this.webdavConfig.depth === "auto" ||
|
||||||
|
this.webdavConfig.depth === "auto_1" ||
|
||||||
|
this.webdavConfig.depth === "auto_infinity" ||
|
||||||
|
this.webdavConfig.depth === "auto_unknown"
|
||||||
|
) {
|
||||||
|
this.webdavConfig.depth = "manual_1";
|
||||||
|
this.webdavConfig.manualRecursive = true;
|
||||||
|
if (this.saveUpdatedConfigFunc !== undefined) {
|
||||||
|
await this.saveUpdatedConfigFunc();
|
||||||
|
console.info(
|
||||||
|
`webdav depth="auto_???" is changed to ${this.webdavConfig.depth}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async walk(): Promise<Entity[]> {
|
||||||
|
await this._init();
|
||||||
|
|
||||||
|
let contents = [] as FileStat[];
|
||||||
|
if (
|
||||||
|
this.webdavConfig.depth === "auto" ||
|
||||||
|
this.webdavConfig.depth === "auto_unknown" ||
|
||||||
|
this.webdavConfig.depth === "auto_1" ||
|
||||||
|
this.webdavConfig.depth === "auto_infinity" /* don't trust auto now */ ||
|
||||||
|
this.webdavConfig.depth === "manual_1"
|
||||||
|
) {
|
||||||
|
// the remote doesn't support infinity propfind,
|
||||||
|
// we need to do a bfs here
|
||||||
|
const q = new Queue([`/${this.remoteBaseDir}`]);
|
||||||
|
const CHUNK_SIZE = 10;
|
||||||
|
while (q.length > 0) {
|
||||||
|
const itemsToFetch: string[] = [];
|
||||||
|
while (q.length > 0) {
|
||||||
|
itemsToFetch.push(q.pop()!);
|
||||||
|
}
|
||||||
|
const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE);
|
||||||
|
// console.debug(itemsToFetchChunks);
|
||||||
|
const subContents = [] as FileStat[];
|
||||||
|
for (const singleChunk of itemsToFetchChunks) {
|
||||||
|
const r = singleChunk.map((x) => {
|
||||||
|
return this.client.getDirectoryContents(x, {
|
||||||
|
deep: false,
|
||||||
|
details: false /* no need for verbose details here */,
|
||||||
|
// TODO: to support .obsidian,
|
||||||
|
// we need to load all files including dot,
|
||||||
|
// anyway to reduce the resources?
|
||||||
|
// glob: "/**" /* avoid dot files by using glob */,
|
||||||
|
}) as Promise<FileStat[]>;
|
||||||
|
});
|
||||||
|
const r2 = flatten(await Promise.all(r));
|
||||||
|
subContents.push(...r2);
|
||||||
|
}
|
||||||
|
for (let i = 0; i < subContents.length; ++i) {
|
||||||
|
const f = subContents[i];
|
||||||
|
contents.push(f);
|
||||||
|
if (f.type === "directory") {
|
||||||
|
q.push(f.filename);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// the remote supports infinity propfind
|
||||||
|
contents = (await this.client.getDirectoryContents(
|
||||||
|
`/${this.remoteBaseDir}`,
|
||||||
|
{
|
||||||
|
deep: true,
|
||||||
|
details: false /* no need for verbose details here */,
|
||||||
|
// TODO: to support .obsidian,
|
||||||
|
// we need to load all files including dot,
|
||||||
|
// anyway to reduce the resources?
|
||||||
|
// glob: "/**" /* avoid dot files by using glob */,
|
||||||
|
}
|
||||||
|
)) as FileStat[];
|
||||||
|
}
|
||||||
|
return contents.map((x) => fromWebdavItemToEntity(x, this.remoteBaseDir));
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(key: string): Promise<Entity> {
|
||||||
|
await this._init();
|
||||||
|
const fullPath = getWebdavPath(key, this.remoteBaseDir);
|
||||||
|
return await this._statFromRoot(fullPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _statFromRoot(key: string): Promise<Entity> {
|
||||||
|
const res = (await this.client.stat(key, {
|
||||||
|
details: false,
|
||||||
|
})) as FileStat;
|
||||||
|
return fromWebdavItemToEntity(res, this.remoteBaseDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
async mkdir(key: string, mtime?: number, ctime?: number): Promise<Entity> {
|
||||||
|
if (!key.endsWith("/")) {
|
||||||
|
throw Error(`you should not call mkdir on ${key}`);
|
||||||
|
}
|
||||||
|
await this._init();
|
||||||
|
const uploadFile = getWebdavPath(key, this.remoteBaseDir);
|
||||||
|
return await this._mkdirFromRoot(uploadFile, mtime, ctime);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _mkdirFromRoot(
|
||||||
|
key: string,
|
||||||
|
mtime?: number,
|
||||||
|
ctime?: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
await this.client.createDirectory(key, {
|
||||||
|
recursive: true,
|
||||||
|
});
|
||||||
|
return await this._statFromRoot(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeFile(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
throw Error(`you should not call writeFile on ${key}`);
|
||||||
|
}
|
||||||
|
await this._init();
|
||||||
|
const uploadFile = getWebdavPath(key, this.remoteBaseDir);
|
||||||
|
return await this._writeFileFromRoot(uploadFile, content, mtime, ctime);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _writeFileFromRoot(
|
||||||
|
key: string,
|
||||||
|
content: ArrayBuffer,
|
||||||
|
mtime: number,
|
||||||
|
ctime: number
|
||||||
|
): Promise<Entity> {
|
||||||
|
await this.client.putFileContents(key, content, {
|
||||||
|
overwrite: true,
|
||||||
|
onUploadProgress: (progress: any) => {
|
||||||
|
console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return await this._statFromRoot(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
async readFile(key: string): Promise<ArrayBuffer> {
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
throw Error(`you should not call readFile on ${key}`);
|
||||||
|
}
|
||||||
|
await this._init();
|
||||||
|
const downloadFile = getWebdavPath(key, this.remoteBaseDir);
|
||||||
|
return await this._readFileFromRoot(downloadFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
async _readFileFromRoot(key: string): Promise<ArrayBuffer> {
|
||||||
|
const buff = (await this.client.getFileContents(key)) as BufferLike;
|
||||||
|
if (buff instanceof ArrayBuffer) {
|
||||||
|
return buff;
|
||||||
|
} else if (buff instanceof Buffer) {
|
||||||
|
return bufferToArrayBuffer(buff);
|
||||||
|
}
|
||||||
|
throw Error(`unexpected file content result with type ${typeof buff}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async rm(key: string): Promise<void> {
|
||||||
|
if (key === "/") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await this._init();
|
||||||
|
try {
|
||||||
|
const remoteFileName = getWebdavPath(key, this.remoteBaseDir);
|
||||||
|
await this.client.deleteFile(remoteFileName);
|
||||||
|
// console.info(`delete ${remoteFileName} succeeded`);
|
||||||
|
} catch (err) {
|
||||||
|
console.error("some error while deleting");
|
||||||
|
console.error(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkConnect(callbackFunc?: any): Promise<boolean> {
|
||||||
|
if (
|
||||||
|
!(
|
||||||
|
this.webdavConfig.address.startsWith("http://") ||
|
||||||
|
this.webdavConfig.address.startsWith("https://")
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
const err =
|
||||||
|
"Error: the url should start with http(s):// but it does not!";
|
||||||
|
console.error(err);
|
||||||
|
if (callbackFunc !== undefined) {
|
||||||
|
callbackFunc(err);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this._init();
|
||||||
|
const results = await this._statFromRoot(`/${this.remoteBaseDir}/`);
|
||||||
|
if (results === undefined) {
|
||||||
|
const err = "results is undefined";
|
||||||
|
console.error(err);
|
||||||
|
callbackFunc?.(err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err);
|
||||||
|
callbackFunc?.(err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getUserDisplayName(): Promise<string> {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
|
||||||
|
async revokeAuth() {
|
||||||
|
throw new Error("Method not implemented.");
|
||||||
|
}
|
||||||
|
}
|
|
@ -7,7 +7,7 @@ import {
|
||||||
RemotelySavePluginSettings,
|
RemotelySavePluginSettings,
|
||||||
QRExportType,
|
QRExportType,
|
||||||
} from "./baseTypes";
|
} from "./baseTypes";
|
||||||
import { getShrinkedSettings } from "./remoteForOnedrive";
|
import { getShrinkedSettings } from "./fsOnedrive";
|
||||||
|
|
||||||
export const exportQrCodeUri = async (
|
export const exportQrCodeUri = async (
|
||||||
settings: RemotelySavePluginSettings,
|
settings: RemotelySavePluginSettings,
|
||||||
|
|
|
@ -133,13 +133,10 @@
|
||||||
"settings_runoncestartup_1sec": "sync once after 1 second of start up",
|
"settings_runoncestartup_1sec": "sync once after 1 second of start up",
|
||||||
"settings_runoncestartup_10sec": "sync once after 10 seconds of start up",
|
"settings_runoncestartup_10sec": "sync once after 10 seconds of start up",
|
||||||
"settings_runoncestartup_30sec": "sync once after 30 seconds of start up",
|
"settings_runoncestartup_30sec": "sync once after 30 seconds of start up",
|
||||||
"settings_saverun": "Sync On Save (experimental)",
|
"settings_synconsave": "Sync On Save (experimental)",
|
||||||
"settings_saverun_desc": "A sync will be triggered if a file save action happened within a few seconds. Please pay attention that syncing is potentially a heavy action and battery may be impacted. (May need to reload the plugin or restart Obsidian after changing)",
|
"settings_synconsave_desc": "If you change your file, the plugin tries to trigger a sync.",
|
||||||
"settings_saverun_notset": "(not set)",
|
"settings_synconsave_disable": "Disable (default)",
|
||||||
"settings_saverun_1sec": "check every 1 second",
|
"settings_synconsave_enable": "Enable",
|
||||||
"settings_saverun_5sec": "check every 5 seconds",
|
|
||||||
"settings_saverun_10sec": "check every 10 seconds (recommended)",
|
|
||||||
"settings_saverun_1min": "check every 1 minute",
|
|
||||||
"settings_skiplargefiles": "Skip Large Files",
|
"settings_skiplargefiles": "Skip Large Files",
|
||||||
"settings_skiplargefiles_desc": "Skip files with sizes larger than the threshold. Here 1 MB = 10^6 bytes.",
|
"settings_skiplargefiles_desc": "Skip files with sizes larger than the threshold. Here 1 MB = 10^6 bytes.",
|
||||||
"settings_skiplargefiles_notset": "(not set)",
|
"settings_skiplargefiles_notset": "(not set)",
|
||||||
|
|
|
@ -132,13 +132,10 @@
|
||||||
"settings_runoncestartup_1sec": "启动后第 1 秒运行一次",
|
"settings_runoncestartup_1sec": "启动后第 1 秒运行一次",
|
||||||
"settings_runoncestartup_10sec": "启动后第 10 秒运行一次",
|
"settings_runoncestartup_10sec": "启动后第 10 秒运行一次",
|
||||||
"settings_runoncestartup_30sec": "启动后第 30 秒运行一次",
|
"settings_runoncestartup_30sec": "启动后第 30 秒运行一次",
|
||||||
"settings_saverun": "保存时同步(实验性质)",
|
"settings_synconsave": "保存时同步(实验性质)",
|
||||||
"settings_saverun_desc": "插件如果检查到当前文件在最近一段时间有修改保存过,则尝试同步。请注意,同步是一个很重的操作,因此会影响到耗电量。(修改设置后可能需要重载插件或重启。)",
|
"settings_synconsave_desc": "插件如果检查到当前文件在最近一段时间有修改保存过,则尝试同步。请注意,同步是一个很重的操作,因此会影响到耗电量。(修改设置后可能需要重载插件或重启。)",
|
||||||
"settings_saverun_notset": "(不设置)",
|
"settings_synconsave_disable": "关闭(默认)",
|
||||||
"settings_saverun_1sec": "隔 1 秒检查一次",
|
"settings_synconsave_enable": "开启",
|
||||||
"settings_saverun_5sec": "隔 5 秒检查一次",
|
|
||||||
"settings_saverun_10sec": "隔 10 秒检查一次(推荐)",
|
|
||||||
"settings_saverun_1min": "隔 1 分钟检查一次",
|
|
||||||
"settings_skiplargefiles": "跳过大文件",
|
"settings_skiplargefiles": "跳过大文件",
|
||||||
"settings_skiplargefiles_desc": "跳过大于某一个阈值的文件。这里 1 MB = 10^6 bytes。",
|
"settings_skiplargefiles_desc": "跳过大于某一个阈值的文件。这里 1 MB = 10^6 bytes。",
|
||||||
"settings_skiplargefiles_notset": "(不设置)",
|
"settings_skiplargefiles_notset": "(不设置)",
|
||||||
|
|
|
@ -131,13 +131,10 @@
|
||||||
"settings_runoncestartup_1sec": "啟動後第 1 秒執行一次",
|
"settings_runoncestartup_1sec": "啟動後第 1 秒執行一次",
|
||||||
"settings_runoncestartup_10sec": "啟動後第 10 秒執行一次",
|
"settings_runoncestartup_10sec": "啟動後第 10 秒執行一次",
|
||||||
"settings_runoncestartup_30sec": "啟動後第 30 秒執行一次",
|
"settings_runoncestartup_30sec": "啟動後第 30 秒執行一次",
|
||||||
"settings_saverun": "儲存時同步(實驗性質)",
|
"settings_synconsave": "儲存時同步(實驗性質)",
|
||||||
"settings_saverun_desc": "外掛如果檢查到當前檔案在最近一段時間有修改儲存過,則嘗試同步。請注意,同步是一個很重的操作,因此會影響到耗電量。(修改設定後可能需要過載外掛或重啟。)",
|
"settings_synconsave_desc": "外掛如果檢查到當前檔案在最近一段時間有修改儲存過,則嘗試同步。請注意,同步是一個很重的操作,因此會影響到耗電量。(修改設定後可能需要過載外掛或重啟。)",
|
||||||
"settings_saverun_notset": "(不設定)",
|
"settings_synconsave_disable": "關閉(預設)",
|
||||||
"settings_saverun_1sec": "隔 1 秒檢查一次",
|
"settings_synconsave_enable": "開啟",
|
||||||
"settings_saverun_5sec": "隔 5 秒檢查一次",
|
|
||||||
"settings_saverun_10sec": "隔 10 秒檢查一次(推薦)",
|
|
||||||
"settings_saverun_1min": "隔 1 分鐘檢查一次",
|
|
||||||
"settings_skiplargefiles": "跳過大檔案",
|
"settings_skiplargefiles": "跳過大檔案",
|
||||||
"settings_skiplargefiles_desc": "跳過大於某一個閾值的檔案。這裡 1 MB = 10^6 bytes。",
|
"settings_skiplargefiles_desc": "跳過大於某一個閾值的檔案。這裡 1 MB = 10^6 bytes。",
|
||||||
"settings_skiplargefiles_notset": "(不設定)",
|
"settings_skiplargefiles_notset": "(不設定)",
|
||||||
|
|
76
src/local.ts
76
src/local.ts
|
@ -1,76 +0,0 @@
|
||||||
import { TFile, TFolder, type Vault } from "obsidian";
|
|
||||||
import type { Entity, MixedEntity } from "./baseTypes";
|
|
||||||
import { listFilesInObsFolder } from "./obsFolderLister";
|
|
||||||
import { Profiler } from "./profiler";
|
|
||||||
|
|
||||||
export const getLocalEntityList = async (
|
|
||||||
vault: Vault,
|
|
||||||
syncConfigDir: boolean,
|
|
||||||
configDir: string,
|
|
||||||
pluginID: string,
|
|
||||||
profiler: Profiler
|
|
||||||
) => {
|
|
||||||
profiler.addIndent();
|
|
||||||
profiler.insert("enter getLocalEntityList");
|
|
||||||
const local: Entity[] = [];
|
|
||||||
|
|
||||||
const localTAbstractFiles = vault.getAllLoadedFiles();
|
|
||||||
profiler.insert("finish getting getAllLoadedFiles");
|
|
||||||
for (const entry of localTAbstractFiles) {
|
|
||||||
let r = {} as Entity;
|
|
||||||
let key = entry.path;
|
|
||||||
|
|
||||||
if (entry.path === "/") {
|
|
||||||
// ignore
|
|
||||||
continue;
|
|
||||||
} else if (entry instanceof TFile) {
|
|
||||||
let mtimeLocal: number | undefined = entry.stat.mtime;
|
|
||||||
if (mtimeLocal <= 0) {
|
|
||||||
mtimeLocal = entry.stat.ctime;
|
|
||||||
}
|
|
||||||
if (mtimeLocal === 0) {
|
|
||||||
mtimeLocal = undefined;
|
|
||||||
}
|
|
||||||
if (mtimeLocal === undefined) {
|
|
||||||
throw Error(
|
|
||||||
`Your file has last modified time 0: ${key}, don't know how to deal with it`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
r = {
|
|
||||||
key: entry.path, // local always unencrypted
|
|
||||||
keyRaw: entry.path,
|
|
||||||
mtimeCli: mtimeLocal,
|
|
||||||
mtimeSvr: mtimeLocal,
|
|
||||||
size: entry.stat.size, // local always unencrypted
|
|
||||||
sizeRaw: entry.stat.size,
|
|
||||||
};
|
|
||||||
} else if (entry instanceof TFolder) {
|
|
||||||
key = `${entry.path}/`;
|
|
||||||
r = {
|
|
||||||
key: key,
|
|
||||||
keyRaw: key,
|
|
||||||
size: 0,
|
|
||||||
sizeRaw: 0,
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
throw Error(`unexpected ${entry}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
local.push(r);
|
|
||||||
}
|
|
||||||
|
|
||||||
profiler.insert("finish transforming getAllLoadedFiles");
|
|
||||||
|
|
||||||
if (syncConfigDir) {
|
|
||||||
profiler.insert("into syncConfigDir");
|
|
||||||
const syncFiles = await listFilesInObsFolder(configDir, vault, pluginID);
|
|
||||||
for (const f of syncFiles) {
|
|
||||||
local.push(f);
|
|
||||||
}
|
|
||||||
profiler.insert("finish syncConfigDir");
|
|
||||||
}
|
|
||||||
|
|
||||||
profiler.insert("finish getLocalEntityList");
|
|
||||||
profiler.removeIndent();
|
|
||||||
return local;
|
|
||||||
};
|
|
|
@ -3,12 +3,10 @@ import { extendPrototype } from "localforage-getitems";
|
||||||
extendPrototype(localforage);
|
extendPrototype(localforage);
|
||||||
export type LocalForage = typeof localforage;
|
export type LocalForage = typeof localforage;
|
||||||
import { nanoid } from "nanoid";
|
import { nanoid } from "nanoid";
|
||||||
import { requireApiVersion, TAbstractFile, TFile, TFolder } from "obsidian";
|
|
||||||
|
|
||||||
import { API_VER_STAT_FOLDER } from "./baseTypes";
|
|
||||||
import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
import type { Entity, MixedEntity, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
||||||
import type { SyncPlanType } from "./sync";
|
import type { SyncPlanType } from "./sync";
|
||||||
import { statFix, toText, unixTimeToStr } from "./misc";
|
import { unixTimeToStr } from "./misc";
|
||||||
|
|
||||||
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220];
|
const DB_VERSION_NUMBER_IN_HISTORY = [20211114, 20220108, 20220326, 20240220];
|
||||||
export const DEFAULT_DB_VERSION_NUMBER: number = 20240220;
|
export const DEFAULT_DB_VERSION_NUMBER: number = 20240220;
|
||||||
|
|
740
src/main.ts
740
src/main.ts
|
@ -7,7 +7,6 @@ import {
|
||||||
setIcon,
|
setIcon,
|
||||||
FileSystemAdapter,
|
FileSystemAdapter,
|
||||||
Platform,
|
Platform,
|
||||||
requestUrl,
|
|
||||||
requireApiVersion,
|
requireApiVersion,
|
||||||
Events,
|
Events,
|
||||||
} from "obsidian";
|
} from "obsidian";
|
||||||
|
@ -26,7 +25,6 @@ import {
|
||||||
} from "./baseTypes";
|
} from "./baseTypes";
|
||||||
import { importQrCodeUri } from "./importExport";
|
import { importQrCodeUri } from "./importExport";
|
||||||
import {
|
import {
|
||||||
insertSyncPlanRecordByVault,
|
|
||||||
prepareDBs,
|
prepareDBs,
|
||||||
InternalDBs,
|
InternalDBs,
|
||||||
clearExpiredSyncPlanRecords,
|
clearExpiredSyncPlanRecords,
|
||||||
|
@ -34,43 +32,35 @@ import {
|
||||||
clearAllLoggerOutputRecords,
|
clearAllLoggerOutputRecords,
|
||||||
upsertLastSuccessSyncTimeByVault,
|
upsertLastSuccessSyncTimeByVault,
|
||||||
getLastSuccessSyncTimeByVault,
|
getLastSuccessSyncTimeByVault,
|
||||||
getAllPrevSyncRecordsByVaultAndProfile,
|
|
||||||
insertProfilerResultByVault,
|
|
||||||
} from "./localdb";
|
} from "./localdb";
|
||||||
import { RemoteClient } from "./remote";
|
|
||||||
import {
|
import {
|
||||||
DEFAULT_DROPBOX_CONFIG,
|
DEFAULT_DROPBOX_CONFIG,
|
||||||
getAuthUrlAndVerifier as getAuthUrlAndVerifierDropbox,
|
|
||||||
sendAuthReq as sendAuthReqDropbox,
|
sendAuthReq as sendAuthReqDropbox,
|
||||||
setConfigBySuccessfullAuthInplace as setConfigBySuccessfullAuthInplaceDropbox,
|
setConfigBySuccessfullAuthInplace as setConfigBySuccessfullAuthInplaceDropbox,
|
||||||
} from "./remoteForDropbox";
|
} from "./fsDropbox";
|
||||||
import {
|
import {
|
||||||
AccessCodeResponseSuccessfulType,
|
AccessCodeResponseSuccessfulType,
|
||||||
DEFAULT_ONEDRIVE_CONFIG,
|
DEFAULT_ONEDRIVE_CONFIG,
|
||||||
sendAuthReq as sendAuthReqOnedrive,
|
sendAuthReq as sendAuthReqOnedrive,
|
||||||
setConfigBySuccessfullAuthInplace as setConfigBySuccessfullAuthInplaceOnedrive,
|
setConfigBySuccessfullAuthInplace as setConfigBySuccessfullAuthInplaceOnedrive,
|
||||||
} from "./remoteForOnedrive";
|
} from "./fsOnedrive";
|
||||||
import { DEFAULT_S3_CONFIG } from "./remoteForS3";
|
import { DEFAULT_S3_CONFIG } from "./fsS3";
|
||||||
import { DEFAULT_WEBDAV_CONFIG } from "./remoteForWebdav";
|
import { DEFAULT_WEBDAV_CONFIG } from "./fsWebdav";
|
||||||
import { RemotelySaveSettingTab } from "./settings";
|
import { RemotelySaveSettingTab } from "./settings";
|
||||||
import {
|
|
||||||
doActualSync,
|
|
||||||
ensembleMixedEnties,
|
|
||||||
getSyncPlanInplace,
|
|
||||||
isPasswordOk,
|
|
||||||
SyncStatusType,
|
|
||||||
} from "./sync";
|
|
||||||
import { messyConfigToNormal, normalConfigToMessy } from "./configPersist";
|
import { messyConfigToNormal, normalConfigToMessy } from "./configPersist";
|
||||||
import { getLocalEntityList } from "./local";
|
|
||||||
import { I18n } from "./i18n";
|
import { I18n } from "./i18n";
|
||||||
import type { LangType, LangTypeAndAuto, TransItemType } from "./i18n";
|
import type { LangTypeAndAuto, TransItemType } from "./i18n";
|
||||||
import { SyncAlgoV3Modal } from "./syncAlgoV3Notice";
|
import { SyncAlgoV3Modal } from "./syncAlgoV3Notice";
|
||||||
|
|
||||||
import AggregateError from "aggregate-error";
|
import AggregateError from "aggregate-error";
|
||||||
import { exportVaultSyncPlansToFiles } from "./debugMode";
|
import { exportVaultSyncPlansToFiles } from "./debugMode";
|
||||||
import { changeMobileStatusBar, compareVersion } from "./misc";
|
import { changeMobileStatusBar } from "./misc";
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
import { Profiler } from "./profiler";
|
import { Profiler } from "./profiler";
|
||||||
|
import { FakeFsLocal } from "./fsLocal";
|
||||||
|
import { FakeFsEncrypt } from "./fsEncrypt";
|
||||||
|
import { syncer } from "./sync";
|
||||||
|
import { getClient } from "./fsGetter";
|
||||||
|
import throttle from "lodash/throttle";
|
||||||
|
|
||||||
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
||||||
s3: DEFAULT_S3_CONFIG,
|
s3: DEFAULT_S3_CONFIG,
|
||||||
|
@ -141,7 +131,8 @@ const getIconSvg = () => {
|
||||||
export default class RemotelySavePlugin extends Plugin {
|
export default class RemotelySavePlugin extends Plugin {
|
||||||
settings!: RemotelySavePluginSettings;
|
settings!: RemotelySavePluginSettings;
|
||||||
db!: InternalDBs;
|
db!: InternalDBs;
|
||||||
syncStatus!: SyncStatusType;
|
isSyncing!: boolean;
|
||||||
|
hasPendingSyncOnSave!: boolean;
|
||||||
statusBarElement!: HTMLSpanElement;
|
statusBarElement!: HTMLSpanElement;
|
||||||
oauth2Info!: OAuth2Info;
|
oauth2Info!: OAuth2Info;
|
||||||
currLogLevel!: string;
|
currLogLevel!: string;
|
||||||
|
@ -156,7 +147,25 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
appContainerObserver?: MutationObserver;
|
appContainerObserver?: MutationObserver;
|
||||||
|
|
||||||
async syncRun(triggerSource: SyncTriggerSourceType = "manual") {
|
async syncRun(triggerSource: SyncTriggerSourceType = "manual") {
|
||||||
const profiler = new Profiler("start of syncRun");
|
const profiler = new Profiler();
|
||||||
|
const fsLocal = new FakeFsLocal(
|
||||||
|
this.app.vault,
|
||||||
|
this.settings.syncConfigDir ?? false,
|
||||||
|
this.app.vault.configDir,
|
||||||
|
this.manifest.id,
|
||||||
|
profiler,
|
||||||
|
this.settings.deleteToWhere ?? "system"
|
||||||
|
);
|
||||||
|
const fsRemote = getClient(
|
||||||
|
this.settings,
|
||||||
|
this.app.vault.getName(),
|
||||||
|
async () => await this.saveSettings()
|
||||||
|
);
|
||||||
|
const fsEncrypt = new FakeFsEncrypt(
|
||||||
|
fsRemote,
|
||||||
|
this.settings.password ?? "",
|
||||||
|
this.settings.encryptionMethod ?? "rclone-base64"
|
||||||
|
);
|
||||||
|
|
||||||
const t = (x: TransItemType, vars?: any) => {
|
const t = (x: TransItemType, vars?: any) => {
|
||||||
return this.i18n.t(x, vars);
|
return this.i18n.t(x, vars);
|
||||||
|
@ -164,333 +173,241 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
|
|
||||||
const profileID = this.getCurrProfileID();
|
const profileID = this.getCurrProfileID();
|
||||||
|
|
||||||
const getNotice = (x: string, timeout?: number) => {
|
const getProtectError = (
|
||||||
// only show notices in manual mode
|
protectModifyPercentage: number,
|
||||||
// no notice in auto mode
|
realModifyDeleteCount: number,
|
||||||
if (triggerSource === "manual" || triggerSource === "dry") {
|
allFilesCount: number
|
||||||
new Notice(x, timeout);
|
) => {
|
||||||
|
const percent = ((100 * realModifyDeleteCount) / allFilesCount).toFixed(
|
||||||
|
1
|
||||||
|
);
|
||||||
|
const res = t("syncrun_abort_protectmodifypercentage", {
|
||||||
|
protectModifyPercentage,
|
||||||
|
realModifyDeleteCount,
|
||||||
|
allFilesCount,
|
||||||
|
percent,
|
||||||
|
});
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getNotice = (
|
||||||
|
s: SyncTriggerSourceType,
|
||||||
|
msg: string,
|
||||||
|
timeout?: number
|
||||||
|
) => {
|
||||||
|
if (s === "manual" || s === "dry") {
|
||||||
|
new Notice(msg, timeout);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if (this.syncStatus !== "idle") {
|
|
||||||
// really, users don't want to see this in auto mode
|
const notifyFunc = async (s: SyncTriggerSourceType, step: number) => {
|
||||||
// so we use getNotice to avoid unnecessary show up
|
switch (step) {
|
||||||
|
case 0:
|
||||||
|
if (s === "dry") {
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
getNotice(s, t("syncrun_shortstep0"));
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step0"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 1:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
getNotice(
|
||||||
|
s,
|
||||||
|
t("syncrun_shortstep1", {
|
||||||
|
serviceType: this.settings.serviceType,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
getNotice(
|
||||||
|
s,
|
||||||
|
t("syncrun_step1", {
|
||||||
|
serviceType: this.settings.serviceType,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 2:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step2"));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 3:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step3"));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 4:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step4"));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 5:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step5"));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 6:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step6"));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 7:
|
||||||
|
if (s === "dry") {
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
getNotice(s, t("syncrun_shortstep2skip"));
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step7skip"));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
// pass
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step7"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 8:
|
||||||
|
if (this.settings.currLogLevel === "info") {
|
||||||
|
getNotice(s, t("syncrun_shortstep2"));
|
||||||
|
} else {
|
||||||
|
getNotice(s, t("syncrun_step8"));
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw Error(`unknown step=${step} for showing notice`);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const errNotifyFunc = async (s: SyncTriggerSourceType, error: Error) => {
|
||||||
|
console.error(error);
|
||||||
|
if (error instanceof AggregateError) {
|
||||||
|
for (const e of error.errors) {
|
||||||
|
getNotice(s, e.message, 10 * 1000);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
getNotice(s, error?.message ?? "error while sync", 10 * 1000);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const ribboonFunc = async (s: SyncTriggerSourceType, step: number) => {
|
||||||
|
if (step === 1) {
|
||||||
|
if (this.syncRibbon !== undefined) {
|
||||||
|
setIcon(this.syncRibbon, iconNameSyncRunning);
|
||||||
|
this.syncRibbon.setAttribute(
|
||||||
|
"aria-label",
|
||||||
|
t("syncrun_syncingribbon", {
|
||||||
|
pluginName: this.manifest.name,
|
||||||
|
triggerSource: s,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else if (step === 8) {
|
||||||
|
// last step
|
||||||
|
if (this.syncRibbon !== undefined) {
|
||||||
|
setIcon(this.syncRibbon, iconNameSyncWait);
|
||||||
|
let originLabel = `${this.manifest.name}`;
|
||||||
|
this.syncRibbon.setAttribute("aria-label", originLabel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const statusBarFunc = async (s: SyncTriggerSourceType, step: number) => {
|
||||||
|
if (step === 1) {
|
||||||
|
// change status to "syncing..." on statusbar
|
||||||
|
this.updateLastSuccessSyncMsg(-1);
|
||||||
|
} else if (step === 8) {
|
||||||
|
const lastSuccessSyncMillis = Date.now();
|
||||||
|
await upsertLastSuccessSyncTimeByVault(
|
||||||
|
this.db,
|
||||||
|
this.vaultRandomID,
|
||||||
|
lastSuccessSyncMillis
|
||||||
|
);
|
||||||
|
this.updateLastSuccessSyncMsg(lastSuccessSyncMillis);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const markIsSyncingFunc = async (isSyncing: boolean) => {
|
||||||
|
this.isSyncing = isSyncing;
|
||||||
|
};
|
||||||
|
|
||||||
|
const callbackSyncProcess = async (
|
||||||
|
realCounter: number,
|
||||||
|
realTotalCount: number,
|
||||||
|
pathName: string,
|
||||||
|
decision: string
|
||||||
|
) => {
|
||||||
|
this.setCurrSyncMsg(
|
||||||
|
realCounter,
|
||||||
|
realTotalCount,
|
||||||
|
pathName,
|
||||||
|
decision,
|
||||||
|
triggerSource
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (this.isSyncing) {
|
||||||
getNotice(
|
getNotice(
|
||||||
|
triggerSource,
|
||||||
t("syncrun_alreadyrunning", {
|
t("syncrun_alreadyrunning", {
|
||||||
pluginName: this.manifest.name,
|
pluginName: this.manifest.name,
|
||||||
syncStatus: this.syncStatus,
|
syncStatus: "running",
|
||||||
newTriggerSource: triggerSource,
|
newTriggerSource: triggerSource,
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
if (this.currSyncMsg !== undefined && this.currSyncMsg !== "") {
|
if (this.currSyncMsg !== undefined && this.currSyncMsg !== "") {
|
||||||
getNotice(this.currSyncMsg);
|
getNotice(triggerSource, this.currSyncMsg);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let originLabel = `${this.manifest.name}`;
|
await syncer(
|
||||||
if (this.syncRibbon !== undefined) {
|
fsLocal,
|
||||||
originLabel = this.syncRibbon.getAttribute("aria-label") as string;
|
fsRemote,
|
||||||
}
|
fsEncrypt,
|
||||||
|
profiler,
|
||||||
try {
|
|
||||||
console.info(
|
|
||||||
`${
|
|
||||||
this.manifest.id
|
|
||||||
}-${Date.now()}: start sync, triggerSource=${triggerSource}`
|
|
||||||
);
|
|
||||||
|
|
||||||
if (this.syncRibbon !== undefined) {
|
|
||||||
setIcon(this.syncRibbon, iconNameSyncRunning);
|
|
||||||
this.syncRibbon.setAttribute(
|
|
||||||
"aria-label",
|
|
||||||
t("syncrun_syncingribbon", {
|
|
||||||
pluginName: this.manifest.name,
|
|
||||||
triggerSource: triggerSource,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (triggerSource === "dry") {
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
getNotice(t("syncrun_shortstep0"));
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step0"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// change status to "syncing..." on statusbar
|
|
||||||
if (this.statusBarElement !== undefined) {
|
|
||||||
this.updateLastSuccessSyncMsg(-1);
|
|
||||||
}
|
|
||||||
//console.info(`huh ${this.settings.password}`)
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
getNotice(
|
|
||||||
t("syncrun_shortstep1", {
|
|
||||||
serviceType: this.settings.serviceType,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
getNotice(
|
|
||||||
t("syncrun_step1", {
|
|
||||||
serviceType: this.settings.serviceType,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.syncStatus = "preparing";
|
|
||||||
profiler.insert("finish step1");
|
|
||||||
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step2"));
|
|
||||||
}
|
|
||||||
this.syncStatus = "getting_remote_files_list";
|
|
||||||
const self = this;
|
|
||||||
const client = new RemoteClient(
|
|
||||||
this.settings.serviceType,
|
|
||||||
this.settings.s3,
|
|
||||||
this.settings.webdav,
|
|
||||||
this.settings.dropbox,
|
|
||||||
this.settings.onedrive,
|
|
||||||
this.app.vault.getName(),
|
|
||||||
() => self.saveSettings(),
|
|
||||||
profiler
|
|
||||||
);
|
|
||||||
const remoteEntityList = await client.listAllFromRemote();
|
|
||||||
console.debug("remoteEntityList:");
|
|
||||||
console.debug(remoteEntityList);
|
|
||||||
|
|
||||||
profiler.insert("finish step2 (listing remote)");
|
|
||||||
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step3"));
|
|
||||||
}
|
|
||||||
this.syncStatus = "checking_password";
|
|
||||||
|
|
||||||
const cipher = new Cipher(
|
|
||||||
this.settings.password,
|
|
||||||
this.settings.encryptionMethod ?? "unknown"
|
|
||||||
);
|
|
||||||
const passwordCheckResult = await isPasswordOk(remoteEntityList, cipher);
|
|
||||||
if (!passwordCheckResult.ok) {
|
|
||||||
getNotice(t("syncrun_passworderr"));
|
|
||||||
throw Error(passwordCheckResult.reason);
|
|
||||||
}
|
|
||||||
|
|
||||||
profiler.insert("finish step3 (checking password)");
|
|
||||||
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step4"));
|
|
||||||
}
|
|
||||||
this.syncStatus = "getting_local_meta";
|
|
||||||
const localEntityList = await getLocalEntityList(
|
|
||||||
this.app.vault,
|
|
||||||
this.settings.syncConfigDir ?? false,
|
|
||||||
this.app.vault.configDir,
|
|
||||||
this.manifest.id,
|
|
||||||
profiler
|
|
||||||
);
|
|
||||||
console.debug("localEntityList:");
|
|
||||||
console.debug(localEntityList);
|
|
||||||
|
|
||||||
profiler.insert("finish step4 (local meta)");
|
|
||||||
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step5"));
|
|
||||||
}
|
|
||||||
this.syncStatus = "getting_local_prev_sync";
|
|
||||||
const prevSyncEntityList = await getAllPrevSyncRecordsByVaultAndProfile(
|
|
||||||
this.db,
|
|
||||||
this.vaultRandomID,
|
|
||||||
profileID
|
|
||||||
);
|
|
||||||
console.debug("prevSyncEntityList:");
|
|
||||||
console.debug(prevSyncEntityList);
|
|
||||||
|
|
||||||
profiler.insert("finish step5 (prev sync)");
|
|
||||||
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step6"));
|
|
||||||
}
|
|
||||||
this.syncStatus = "generating_plan";
|
|
||||||
let mixedEntityMappings = await ensembleMixedEnties(
|
|
||||||
localEntityList,
|
|
||||||
prevSyncEntityList,
|
|
||||||
remoteEntityList,
|
|
||||||
this.settings.syncConfigDir ?? false,
|
|
||||||
this.app.vault.configDir,
|
|
||||||
this.settings.syncUnderscoreItems ?? false,
|
|
||||||
this.settings.ignorePaths ?? [],
|
|
||||||
cipher,
|
|
||||||
this.settings.serviceType,
|
|
||||||
profiler
|
|
||||||
);
|
|
||||||
profiler.insert("finish building partial mixedEntity");
|
|
||||||
mixedEntityMappings = await getSyncPlanInplace(
|
|
||||||
mixedEntityMappings,
|
|
||||||
this.settings.howToCleanEmptyFolder ?? "skip",
|
|
||||||
this.settings.skipSizeLargerThan ?? -1,
|
|
||||||
this.settings.conflictAction ?? "keep_newer",
|
|
||||||
this.settings.syncDirection ?? "bidirectional",
|
|
||||||
profiler
|
|
||||||
);
|
|
||||||
console.info(`mixedEntityMappings:`);
|
|
||||||
console.info(mixedEntityMappings); // for debugging
|
|
||||||
profiler.insert("finish building full sync plan");
|
|
||||||
await insertSyncPlanRecordByVault(
|
|
||||||
this.db,
|
|
||||||
mixedEntityMappings,
|
|
||||||
this.vaultRandomID,
|
|
||||||
client.serviceType
|
|
||||||
);
|
|
||||||
|
|
||||||
profiler.insert("finish writing sync plan");
|
|
||||||
profiler.insert("finish step6 (plan)");
|
|
||||||
|
|
||||||
// The operations above are almost read only and kind of safe.
|
|
||||||
// The operations below begins to write or delete (!!!) something.
|
|
||||||
|
|
||||||
if (triggerSource !== "dry") {
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step7"));
|
|
||||||
}
|
|
||||||
this.syncStatus = "syncing";
|
|
||||||
await doActualSync(
|
|
||||||
mixedEntityMappings,
|
|
||||||
client,
|
|
||||||
this.vaultRandomID,
|
|
||||||
profileID,
|
|
||||||
this.app.vault,
|
|
||||||
cipher,
|
|
||||||
this.settings.concurrency ?? 5,
|
|
||||||
(key: string) => self.trash(key),
|
|
||||||
this.settings.protectModifyPercentage ?? 50,
|
|
||||||
(
|
|
||||||
protectModifyPercentage: number,
|
|
||||||
realModifyDeleteCount: number,
|
|
||||||
allFilesCount: number
|
|
||||||
) => {
|
|
||||||
const percent = (
|
|
||||||
(100 * realModifyDeleteCount) /
|
|
||||||
allFilesCount
|
|
||||||
).toFixed(1);
|
|
||||||
const res = t("syncrun_abort_protectmodifypercentage", {
|
|
||||||
protectModifyPercentage,
|
|
||||||
realModifyDeleteCount,
|
|
||||||
allFilesCount,
|
|
||||||
percent,
|
|
||||||
});
|
|
||||||
return res;
|
|
||||||
},
|
|
||||||
(
|
|
||||||
realCounter: number,
|
|
||||||
realTotalCount: number,
|
|
||||||
pathName: string,
|
|
||||||
decision: string
|
|
||||||
) =>
|
|
||||||
self.setCurrSyncMsg(
|
|
||||||
realCounter,
|
|
||||||
realTotalCount,
|
|
||||||
pathName,
|
|
||||||
decision,
|
|
||||||
triggerSource
|
|
||||||
),
|
|
||||||
this.db,
|
|
||||||
profiler
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
this.syncStatus = "syncing";
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
getNotice(t("syncrun_shortstep2skip"));
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step7skip"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cipher.closeResources();
|
|
||||||
|
|
||||||
profiler.insert("finish step7 (actual sync)");
|
|
||||||
|
|
||||||
if (this.settings.currLogLevel === "info") {
|
|
||||||
getNotice(t("syncrun_shortstep2"));
|
|
||||||
} else {
|
|
||||||
getNotice(t("syncrun_step8"));
|
|
||||||
}
|
|
||||||
|
|
||||||
this.syncStatus = "finish";
|
|
||||||
this.syncStatus = "idle";
|
|
||||||
|
|
||||||
profiler.insert("finish step8");
|
|
||||||
|
|
||||||
const lastSuccessSyncMillis = Date.now();
|
|
||||||
await upsertLastSuccessSyncTimeByVault(
|
|
||||||
this.db,
|
|
||||||
this.vaultRandomID,
|
|
||||||
lastSuccessSyncMillis
|
|
||||||
);
|
|
||||||
|
|
||||||
if (this.syncRibbon !== undefined) {
|
|
||||||
setIcon(this.syncRibbon, iconNameSyncWait);
|
|
||||||
this.syncRibbon.setAttribute("aria-label", originLabel);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.statusBarElement !== undefined) {
|
|
||||||
this.updateLastSuccessSyncMsg(lastSuccessSyncMillis);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.syncEvent?.trigger("SYNC_DONE");
|
|
||||||
console.info(
|
|
||||||
`${
|
|
||||||
this.manifest.id
|
|
||||||
}-${Date.now()}: finish sync, triggerSource=${triggerSource}`
|
|
||||||
);
|
|
||||||
} catch (error: any) {
|
|
||||||
profiler.insert("start error branch");
|
|
||||||
const msg = t("syncrun_abort", {
|
|
||||||
manifestID: this.manifest.id,
|
|
||||||
theDate: `${Date.now()}`,
|
|
||||||
triggerSource: triggerSource,
|
|
||||||
syncStatus: this.syncStatus,
|
|
||||||
});
|
|
||||||
console.error(msg);
|
|
||||||
console.error(error);
|
|
||||||
getNotice(msg, 10 * 1000);
|
|
||||||
if (error instanceof AggregateError) {
|
|
||||||
for (const e of error.errors) {
|
|
||||||
getNotice(e.message, 10 * 1000);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
getNotice(error?.message ?? "error while sync", 10 * 1000);
|
|
||||||
}
|
|
||||||
this.syncStatus = "idle";
|
|
||||||
if (this.syncRibbon !== undefined) {
|
|
||||||
setIcon(this.syncRibbon, iconNameSyncWait);
|
|
||||||
this.syncRibbon.setAttribute("aria-label", originLabel);
|
|
||||||
}
|
|
||||||
|
|
||||||
profiler.insert("finish error branch");
|
|
||||||
}
|
|
||||||
|
|
||||||
profiler.insert("finish syncRun");
|
|
||||||
console.debug(profiler.toString());
|
|
||||||
insertProfilerResultByVault(
|
|
||||||
this.db,
|
this.db,
|
||||||
profiler.toString(),
|
triggerSource,
|
||||||
|
profileID,
|
||||||
this.vaultRandomID,
|
this.vaultRandomID,
|
||||||
this.settings.serviceType
|
this.app.vault.configDir,
|
||||||
|
this.settings,
|
||||||
|
getProtectError,
|
||||||
|
markIsSyncingFunc,
|
||||||
|
notifyFunc,
|
||||||
|
errNotifyFunc,
|
||||||
|
ribboonFunc,
|
||||||
|
statusBarFunc,
|
||||||
|
callbackSyncProcess
|
||||||
);
|
);
|
||||||
|
|
||||||
|
fsEncrypt.closeResources();
|
||||||
profiler.clear();
|
profiler.clear();
|
||||||
|
|
||||||
|
this.syncEvent?.trigger("SYNC_DONE");
|
||||||
}
|
}
|
||||||
|
|
||||||
async onload() {
|
async onload() {
|
||||||
|
@ -511,6 +428,8 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
}; // init
|
}; // init
|
||||||
|
|
||||||
this.currSyncMsg = "";
|
this.currSyncMsg = "";
|
||||||
|
this.isSyncing = false;
|
||||||
|
this.hasPendingSyncOnSave = false;
|
||||||
|
|
||||||
this.syncEvent = new Events();
|
this.syncEvent = new Events();
|
||||||
|
|
||||||
|
@ -561,8 +480,6 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
// must AFTER preparing DB
|
// must AFTER preparing DB
|
||||||
this.enableAutoClearSyncPlanHist();
|
this.enableAutoClearSyncPlanHist();
|
||||||
|
|
||||||
this.syncStatus = "idle";
|
|
||||||
|
|
||||||
this.registerObsidianProtocolHandler(COMMAND_URI, async (inputParams) => {
|
this.registerObsidianProtocolHandler(COMMAND_URI, async (inputParams) => {
|
||||||
// console.debug(inputParams);
|
// console.debug(inputParams);
|
||||||
const parsed = importQrCodeUri(inputParams, this.app.vault.getName());
|
const parsed = importQrCodeUri(inputParams, this.app.vault.getName());
|
||||||
|
@ -633,17 +550,12 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
() => self.saveSettings()
|
() => self.saveSettings()
|
||||||
);
|
);
|
||||||
|
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"dropbox",
|
this.settings,
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.settings.dropbox,
|
|
||||||
undefined,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.saveSettings()
|
() => self.saveSettings()
|
||||||
);
|
);
|
||||||
|
const username = await client.getUserDisplayName();
|
||||||
const username = await client.getUser();
|
|
||||||
this.settings.dropbox.username = username;
|
this.settings.dropbox.username = username;
|
||||||
await this.saveSettings();
|
await this.saveSettings();
|
||||||
|
|
||||||
|
@ -729,16 +641,12 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
() => self.saveSettings()
|
() => self.saveSettings()
|
||||||
);
|
);
|
||||||
|
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"onedrive",
|
this.settings,
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.settings.onedrive,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.saveSettings()
|
() => self.saveSettings()
|
||||||
);
|
);
|
||||||
this.settings.onedrive.username = await client.getUser();
|
this.settings.onedrive.username = await client.getUserDisplayName();
|
||||||
await this.saveSettings();
|
await this.saveSettings();
|
||||||
|
|
||||||
this.oauth2Info.verifier = ""; // reset it
|
this.oauth2Info.verifier = ""; // reset it
|
||||||
|
@ -879,7 +787,7 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
} else {
|
} else {
|
||||||
this.enableAutoSyncIfSet();
|
this.enableAutoSyncIfSet();
|
||||||
this.enableInitSyncIfSet();
|
this.enableInitSyncIfSet();
|
||||||
this.enableSyncOnSaveIfSet();
|
this.toggleSyncOnSaveIfSet();
|
||||||
}
|
}
|
||||||
|
|
||||||
// compare versions and read new versions
|
// compare versions and read new versions
|
||||||
|
@ -1191,75 +1099,89 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
enableSyncOnSaveIfSet() {
|
async _checkCurrFileModified(caller: "SYNC" | "FILE_CHANGES") {
|
||||||
|
console.debug(`inside checkCurrFileModified`);
|
||||||
|
const currentFile = this.app.workspace.getActiveFile();
|
||||||
|
|
||||||
|
if (currentFile) {
|
||||||
|
console.debug(`we have currentFile=${currentFile.path}`);
|
||||||
|
// get the last modified time of the current file
|
||||||
|
// if it has modified after lastSuccessSync
|
||||||
|
// then schedule a run for syncOnSaveAfterMilliseconds after it was modified
|
||||||
|
const lastModified = currentFile.stat.mtime;
|
||||||
|
const lastSuccessSyncMillis = await getLastSuccessSyncTimeByVault(
|
||||||
|
this.db,
|
||||||
|
this.vaultRandomID
|
||||||
|
);
|
||||||
|
|
||||||
|
console.debug(
|
||||||
|
`lastModified=${lastModified}, lastSuccessSyncMillis=${lastSuccessSyncMillis}`
|
||||||
|
);
|
||||||
|
|
||||||
|
if (
|
||||||
|
caller === "SYNC" ||
|
||||||
|
(caller === "FILE_CHANGES" && lastModified > lastSuccessSyncMillis)
|
||||||
|
) {
|
||||||
|
console.debug(
|
||||||
|
`so lastModified > lastSuccessSyncMillis or it's called while syncing before`
|
||||||
|
);
|
||||||
|
console.debug(
|
||||||
|
`caller=${caller}, isSyncing=${this.isSyncing}, hasPendingSyncOnSave=${this.hasPendingSyncOnSave}`
|
||||||
|
);
|
||||||
|
if (this.isSyncing) {
|
||||||
|
this.hasPendingSyncOnSave = true;
|
||||||
|
// wait for next event
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
if (this.hasPendingSyncOnSave || caller === "FILE_CHANGES") {
|
||||||
|
this.hasPendingSyncOnSave = false;
|
||||||
|
await this.syncRun("auto_sync_on_save");
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.debug(`no currentFile here`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_syncOnSaveEvent1 = () => {
|
||||||
|
this._checkCurrFileModified("SYNC");
|
||||||
|
};
|
||||||
|
|
||||||
|
_syncOnSaveEvent2 = throttle(
|
||||||
|
async () => {
|
||||||
|
await this._checkCurrFileModified("FILE_CHANGES");
|
||||||
|
},
|
||||||
|
1000 * 3,
|
||||||
|
{
|
||||||
|
leading: false,
|
||||||
|
trailing: true,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
toggleSyncOnSaveIfSet() {
|
||||||
if (
|
if (
|
||||||
this.settings.syncOnSaveAfterMilliseconds !== undefined &&
|
this.settings.syncOnSaveAfterMilliseconds !== undefined &&
|
||||||
this.settings.syncOnSaveAfterMilliseconds !== null &&
|
this.settings.syncOnSaveAfterMilliseconds !== null &&
|
||||||
this.settings.syncOnSaveAfterMilliseconds > 0
|
this.settings.syncOnSaveAfterMilliseconds > 0
|
||||||
) {
|
) {
|
||||||
let runScheduled = false;
|
|
||||||
let needToRunAgain = false;
|
|
||||||
|
|
||||||
const scheduleSyncOnSave = (scheduleTimeFromNow: number) => {
|
|
||||||
console.info(
|
|
||||||
`schedule a run for ${scheduleTimeFromNow} milliseconds later`
|
|
||||||
);
|
|
||||||
runScheduled = true;
|
|
||||||
setTimeout(() => {
|
|
||||||
this.syncRun("auto_sync_on_save");
|
|
||||||
runScheduled = false;
|
|
||||||
}, scheduleTimeFromNow);
|
|
||||||
};
|
|
||||||
|
|
||||||
const checkCurrFileModified = async (caller: "SYNC" | "FILE_CHANGES") => {
|
|
||||||
const currentFile = this.app.workspace.getActiveFile();
|
|
||||||
|
|
||||||
if (currentFile) {
|
|
||||||
// get the last modified time of the current file
|
|
||||||
// if it has modified after lastSuccessSync
|
|
||||||
// then schedule a run for syncOnSaveAfterMilliseconds after it was modified
|
|
||||||
const lastModified = currentFile.stat.mtime;
|
|
||||||
const lastSuccessSyncMillis = await getLastSuccessSyncTimeByVault(
|
|
||||||
this.db,
|
|
||||||
this.vaultRandomID
|
|
||||||
);
|
|
||||||
if (
|
|
||||||
this.syncStatus === "idle" &&
|
|
||||||
lastModified > lastSuccessSyncMillis &&
|
|
||||||
!runScheduled
|
|
||||||
) {
|
|
||||||
scheduleSyncOnSave(this.settings!.syncOnSaveAfterMilliseconds!);
|
|
||||||
} else if (
|
|
||||||
this.syncStatus === "idle" &&
|
|
||||||
needToRunAgain &&
|
|
||||||
!runScheduled
|
|
||||||
) {
|
|
||||||
scheduleSyncOnSave(this.settings!.syncOnSaveAfterMilliseconds!);
|
|
||||||
needToRunAgain = false;
|
|
||||||
} else {
|
|
||||||
if (caller === "FILE_CHANGES") {
|
|
||||||
needToRunAgain = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
this.app.workspace.onLayoutReady(() => {
|
this.app.workspace.onLayoutReady(() => {
|
||||||
// listen to sync done
|
// listen to sync done
|
||||||
this.registerEvent(
|
this.registerEvent(
|
||||||
this.syncEvent?.on("SYNC_DONE", () => {
|
this.syncEvent?.on("SYNC_DONE", this._syncOnSaveEvent1)!
|
||||||
checkCurrFileModified("SYNC");
|
|
||||||
})!
|
|
||||||
);
|
);
|
||||||
|
|
||||||
// listen to current file save changes
|
// listen to current file save changes
|
||||||
this.registerEvent(
|
this.registerEvent(this.app.vault.on("modify", this._syncOnSaveEvent2));
|
||||||
this.app.vault.on("modify", (x) => {
|
this.registerEvent(this.app.vault.on("create", this._syncOnSaveEvent2));
|
||||||
// console.debug(`event=modify! file=${x}`);
|
this.registerEvent(this.app.vault.on("delete", this._syncOnSaveEvent2));
|
||||||
checkCurrFileModified("FILE_CHANGES");
|
|
||||||
})
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
} else {
|
||||||
|
this.syncEvent?.off("SYNC_DONE", this._syncOnSaveEvent1);
|
||||||
|
this.app.vault.off("modify", this._syncOnSaveEvent2);
|
||||||
|
this.app.vault.off("create", this._syncOnSaveEvent2);
|
||||||
|
this.app.vault.off("delete", this._syncOnSaveEvent2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1276,7 +1198,7 @@ export default class RemotelySavePlugin extends Plugin {
|
||||||
await this.saveSettings();
|
await this.saveSettings();
|
||||||
}
|
}
|
||||||
|
|
||||||
async setCurrSyncMsg(
|
setCurrSyncMsg(
|
||||||
i: number,
|
i: number,
|
||||||
totalCount: number,
|
totalCount: number,
|
||||||
pathName: string,
|
pathName: string,
|
||||||
|
|
|
@ -31,13 +31,6 @@ const isLikelyPluginSubFiles = (x: string) => {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const isInsideObsFolder = (x: string, configDir: string) => {
|
|
||||||
if (!configDir.startsWith(".")) {
|
|
||||||
throw Error(`configDir should starts with . but we get ${configDir}`);
|
|
||||||
}
|
|
||||||
return x === configDir || x.startsWith(`${configDir}/`);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listFilesInObsFolder = async (
|
export const listFilesInObsFolder = async (
|
||||||
configDir: string,
|
configDir: string,
|
||||||
vault: Vault,
|
vault: Vault,
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
import { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
||||||
|
import { InternalDBs, insertProfilerResultByVault } from "./localdb";
|
||||||
import { unixTimeToStr } from "./misc";
|
import { unixTimeToStr } from "./misc";
|
||||||
|
|
||||||
interface BreakPoint {
|
interface BreakPoint {
|
||||||
|
@ -79,4 +81,17 @@ export class Profiler {
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async save(
|
||||||
|
db: InternalDBs,
|
||||||
|
vaultRandomID: string,
|
||||||
|
remoteType: SUPPORTED_SERVICES_TYPE
|
||||||
|
) {
|
||||||
|
await insertProfilerResultByVault(
|
||||||
|
db,
|
||||||
|
this.toString(),
|
||||||
|
vaultRandomID,
|
||||||
|
remoteType
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
320
src/remote.ts
320
src/remote.ts
|
@ -1,320 +0,0 @@
|
||||||
import { Vault } from "obsidian";
|
|
||||||
import type {
|
|
||||||
Entity,
|
|
||||||
DropboxConfig,
|
|
||||||
OnedriveConfig,
|
|
||||||
S3Config,
|
|
||||||
SUPPORTED_SERVICES_TYPE,
|
|
||||||
WebdavConfig,
|
|
||||||
UploadedType,
|
|
||||||
} from "./baseTypes";
|
|
||||||
import * as dropbox from "./remoteForDropbox";
|
|
||||||
import * as onedrive from "./remoteForOnedrive";
|
|
||||||
import * as s3 from "./remoteForS3";
|
|
||||||
import * as webdav from "./remoteForWebdav";
|
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
import { Profiler } from "./profiler";
|
|
||||||
|
|
||||||
export class RemoteClient {
|
|
||||||
readonly serviceType: SUPPORTED_SERVICES_TYPE;
|
|
||||||
readonly s3Config?: S3Config;
|
|
||||||
readonly webdavClient?: webdav.WrappedWebdavClient;
|
|
||||||
readonly webdavConfig?: WebdavConfig;
|
|
||||||
readonly dropboxClient?: dropbox.WrappedDropboxClient;
|
|
||||||
readonly dropboxConfig?: DropboxConfig;
|
|
||||||
readonly onedriveClient?: onedrive.WrappedOnedriveClient;
|
|
||||||
readonly onedriveConfig?: OnedriveConfig;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
serviceType: SUPPORTED_SERVICES_TYPE,
|
|
||||||
s3Config?: S3Config,
|
|
||||||
webdavConfig?: WebdavConfig,
|
|
||||||
dropboxConfig?: DropboxConfig,
|
|
||||||
onedriveConfig?: OnedriveConfig,
|
|
||||||
vaultName?: string,
|
|
||||||
saveUpdatedConfigFunc?: () => Promise<any>,
|
|
||||||
profiler?: Profiler
|
|
||||||
) {
|
|
||||||
this.serviceType = serviceType;
|
|
||||||
// the client may modify the config inplace,
|
|
||||||
// so we use a ref not copy of config here
|
|
||||||
if (serviceType === "s3") {
|
|
||||||
this.s3Config = s3Config;
|
|
||||||
} else if (serviceType === "webdav") {
|
|
||||||
if (vaultName === undefined || saveUpdatedConfigFunc === undefined) {
|
|
||||||
throw Error(
|
|
||||||
"remember to provide vault name and callback while init webdav client"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const remoteBaseDir = webdavConfig!.remoteBaseDir || vaultName;
|
|
||||||
this.webdavConfig = webdavConfig;
|
|
||||||
this.webdavClient = webdav.getWebdavClient(
|
|
||||||
this.webdavConfig!,
|
|
||||||
remoteBaseDir,
|
|
||||||
saveUpdatedConfigFunc
|
|
||||||
);
|
|
||||||
} else if (serviceType === "dropbox") {
|
|
||||||
if (vaultName === undefined || saveUpdatedConfigFunc === undefined) {
|
|
||||||
throw Error(
|
|
||||||
"remember to provide vault name and callback while init dropbox client"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const remoteBaseDir = dropboxConfig!.remoteBaseDir || vaultName;
|
|
||||||
this.dropboxConfig = dropboxConfig;
|
|
||||||
this.dropboxClient = dropbox.getDropboxClient(
|
|
||||||
this.dropboxConfig!,
|
|
||||||
remoteBaseDir,
|
|
||||||
saveUpdatedConfigFunc
|
|
||||||
);
|
|
||||||
} else if (serviceType === "onedrive") {
|
|
||||||
if (vaultName === undefined || saveUpdatedConfigFunc === undefined) {
|
|
||||||
throw Error(
|
|
||||||
"remember to provide vault name and callback while init onedrive client"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const remoteBaseDir = onedriveConfig!.remoteBaseDir || vaultName;
|
|
||||||
this.onedriveConfig = onedriveConfig;
|
|
||||||
this.onedriveClient = onedrive.getOnedriveClient(
|
|
||||||
this.onedriveConfig!,
|
|
||||||
remoteBaseDir,
|
|
||||||
saveUpdatedConfigFunc
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
getRemoteMeta = async (fileOrFolderPath: string) => {
|
|
||||||
if (this.serviceType === "s3") {
|
|
||||||
return await s3.getRemoteMeta(
|
|
||||||
s3.getS3Client(this.s3Config!),
|
|
||||||
this.s3Config!,
|
|
||||||
fileOrFolderPath
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "webdav") {
|
|
||||||
return await webdav.getRemoteMeta(this.webdavClient!, fileOrFolderPath);
|
|
||||||
} else if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.getRemoteMeta(this.dropboxClient!, fileOrFolderPath);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.getRemoteMeta(
|
|
||||||
this.onedriveClient!,
|
|
||||||
fileOrFolderPath
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
uploadToRemote = async (
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault | undefined,
|
|
||||||
isRecursively: boolean,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
foldersCreatedBefore: Set<string> | undefined = undefined,
|
|
||||||
uploadRaw: boolean = false,
|
|
||||||
rawContent: string | ArrayBuffer = ""
|
|
||||||
): Promise<UploadedType> => {
|
|
||||||
if (this.serviceType === "s3") {
|
|
||||||
return await s3.uploadToRemote(
|
|
||||||
s3.getS3Client(this.s3Config!),
|
|
||||||
this.s3Config!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
isRecursively,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
uploadRaw,
|
|
||||||
rawContent
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "webdav") {
|
|
||||||
return await webdav.uploadToRemote(
|
|
||||||
this.webdavClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
isRecursively,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
uploadRaw,
|
|
||||||
rawContent
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.uploadToRemote(
|
|
||||||
this.dropboxClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
isRecursively,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
foldersCreatedBefore,
|
|
||||||
uploadRaw,
|
|
||||||
rawContent
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.uploadToRemote(
|
|
||||||
this.onedriveClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
isRecursively,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
foldersCreatedBefore,
|
|
||||||
uploadRaw,
|
|
||||||
rawContent
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
listAllFromRemote = async (): Promise<Entity[]> => {
|
|
||||||
if (this.serviceType === "s3") {
|
|
||||||
return await s3.listAllFromRemote(
|
|
||||||
s3.getS3Client(this.s3Config!),
|
|
||||||
this.s3Config!
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "webdav") {
|
|
||||||
return await webdav.listAllFromRemote(this.webdavClient!);
|
|
||||||
} else if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.listAllFromRemote(this.dropboxClient!);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.listAllFromRemote(this.onedriveClient!);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
downloadFromRemote = async (
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault,
|
|
||||||
mtime: number,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
skipSaving: boolean = false
|
|
||||||
) => {
|
|
||||||
if (this.serviceType === "s3") {
|
|
||||||
return await s3.downloadFromRemote(
|
|
||||||
s3.getS3Client(this.s3Config!),
|
|
||||||
this.s3Config!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
mtime,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
skipSaving
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "webdav") {
|
|
||||||
return await webdav.downloadFromRemote(
|
|
||||||
this.webdavClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
mtime,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
skipSaving
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.downloadFromRemote(
|
|
||||||
this.dropboxClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
mtime,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
skipSaving
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.downloadFromRemote(
|
|
||||||
this.onedriveClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
vault,
|
|
||||||
mtime,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
skipSaving
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
deleteFromRemote = async (
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
synthesizedFolder: boolean = false
|
|
||||||
) => {
|
|
||||||
if (this.serviceType === "s3") {
|
|
||||||
return await s3.deleteFromRemote(
|
|
||||||
s3.getS3Client(this.s3Config!),
|
|
||||||
this.s3Config!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey,
|
|
||||||
synthesizedFolder
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "webdav") {
|
|
||||||
return await webdav.deleteFromRemote(
|
|
||||||
this.webdavClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.deleteFromRemote(
|
|
||||||
this.dropboxClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.deleteFromRemote(
|
|
||||||
this.onedriveClient!,
|
|
||||||
fileOrFolderPath,
|
|
||||||
cipher,
|
|
||||||
remoteEncryptedKey
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
checkConnectivity = async (callbackFunc?: any) => {
|
|
||||||
if (this.serviceType === "s3") {
|
|
||||||
return await s3.checkConnectivity(
|
|
||||||
s3.getS3Client(this.s3Config!),
|
|
||||||
this.s3Config!,
|
|
||||||
callbackFunc
|
|
||||||
);
|
|
||||||
} else if (this.serviceType === "webdav") {
|
|
||||||
return await webdav.checkConnectivity(this.webdavClient!, callbackFunc);
|
|
||||||
} else if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.checkConnectivity(this.dropboxClient!, callbackFunc);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.checkConnectivity(
|
|
||||||
this.onedriveClient!,
|
|
||||||
callbackFunc
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
getUser = async () => {
|
|
||||||
if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.getUserDisplayName(this.dropboxClient!);
|
|
||||||
} else if (this.serviceType === "onedrive") {
|
|
||||||
return await onedrive.getUserDisplayName(this.onedriveClient!);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
revokeAuth = async () => {
|
|
||||||
if (this.serviceType === "dropbox") {
|
|
||||||
return await dropbox.revokeAuth(this.dropboxClient!);
|
|
||||||
} else {
|
|
||||||
throw Error(`not supported service type ${this.serviceType}`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
|
@ -1,844 +0,0 @@
|
||||||
import type { _Object } from "@aws-sdk/client-s3";
|
|
||||||
import {
|
|
||||||
DeleteObjectCommand,
|
|
||||||
GetObjectCommand,
|
|
||||||
HeadBucketCommand,
|
|
||||||
HeadObjectCommand,
|
|
||||||
HeadObjectCommandOutput,
|
|
||||||
ListObjectsV2Command,
|
|
||||||
ListObjectsV2CommandInput,
|
|
||||||
PutObjectCommand,
|
|
||||||
S3Client,
|
|
||||||
} from "@aws-sdk/client-s3";
|
|
||||||
import { Upload } from "@aws-sdk/lib-storage";
|
|
||||||
import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http";
|
|
||||||
import {
|
|
||||||
FetchHttpHandler,
|
|
||||||
FetchHttpHandlerOptions,
|
|
||||||
} from "@smithy/fetch-http-handler";
|
|
||||||
// @ts-ignore
|
|
||||||
import { requestTimeout } from "@smithy/fetch-http-handler/dist-es/request-timeout";
|
|
||||||
import { buildQueryString } from "@smithy/querystring-builder";
|
|
||||||
import { HeaderBag, HttpHandlerOptions, Provider } from "@aws-sdk/types";
|
|
||||||
import { Buffer } from "buffer";
|
|
||||||
import * as mime from "mime-types";
|
|
||||||
import { Vault, requestUrl, RequestUrlParam, Platform } from "obsidian";
|
|
||||||
import { Readable } from "stream";
|
|
||||||
import * as path from "path";
|
|
||||||
import AggregateError from "aggregate-error";
|
|
||||||
import {
|
|
||||||
DEFAULT_CONTENT_TYPE,
|
|
||||||
Entity,
|
|
||||||
S3Config,
|
|
||||||
UploadedType,
|
|
||||||
VALID_REQURL,
|
|
||||||
} from "./baseTypes";
|
|
||||||
import {
|
|
||||||
arrayBufferToBuffer,
|
|
||||||
bufferToArrayBuffer,
|
|
||||||
mkdirpInVault,
|
|
||||||
} from "./misc";
|
|
||||||
|
|
||||||
export { S3Client } from "@aws-sdk/client-s3";
|
|
||||||
|
|
||||||
import PQueue from "p-queue";
|
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// special handler using Obsidian requestUrl
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This is close to origin implementation of FetchHttpHandler
|
|
||||||
* https://github.com/aws/aws-sdk-js-v3/blob/main/packages/fetch-http-handler/src/fetch-http-handler.ts
|
|
||||||
* that is released under Apache 2 License.
|
|
||||||
* But this uses Obsidian requestUrl instead.
|
|
||||||
*/
|
|
||||||
class ObsHttpHandler extends FetchHttpHandler {
|
|
||||||
requestTimeoutInMs: number | undefined;
|
|
||||||
reverseProxyNoSignUrl: string | undefined;
|
|
||||||
constructor(
|
|
||||||
options?: FetchHttpHandlerOptions,
|
|
||||||
reverseProxyNoSignUrl?: string
|
|
||||||
) {
|
|
||||||
super(options);
|
|
||||||
this.requestTimeoutInMs =
|
|
||||||
options === undefined ? undefined : options.requestTimeout;
|
|
||||||
this.reverseProxyNoSignUrl = reverseProxyNoSignUrl;
|
|
||||||
}
|
|
||||||
async handle(
|
|
||||||
request: HttpRequest,
|
|
||||||
{ abortSignal }: HttpHandlerOptions = {}
|
|
||||||
): Promise<{ response: HttpResponse }> {
|
|
||||||
if (abortSignal?.aborted) {
|
|
||||||
const abortError = new Error("Request aborted");
|
|
||||||
abortError.name = "AbortError";
|
|
||||||
return Promise.reject(abortError);
|
|
||||||
}
|
|
||||||
|
|
||||||
let path = request.path;
|
|
||||||
if (request.query) {
|
|
||||||
const queryString = buildQueryString(request.query);
|
|
||||||
if (queryString) {
|
|
||||||
path += `?${queryString}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const { port, method } = request;
|
|
||||||
let url = `${request.protocol}//${request.hostname}${
|
|
||||||
port ? `:${port}` : ""
|
|
||||||
}${path}`;
|
|
||||||
if (
|
|
||||||
this.reverseProxyNoSignUrl !== undefined &&
|
|
||||||
this.reverseProxyNoSignUrl !== ""
|
|
||||||
) {
|
|
||||||
const urlObj = new URL(url);
|
|
||||||
urlObj.host = this.reverseProxyNoSignUrl;
|
|
||||||
url = urlObj.href;
|
|
||||||
}
|
|
||||||
const body =
|
|
||||||
method === "GET" || method === "HEAD" ? undefined : request.body;
|
|
||||||
|
|
||||||
const transformedHeaders: Record<string, string> = {};
|
|
||||||
for (const key of Object.keys(request.headers)) {
|
|
||||||
const keyLower = key.toLowerCase();
|
|
||||||
if (keyLower === "host" || keyLower === "content-length") {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
transformedHeaders[keyLower] = request.headers[key];
|
|
||||||
}
|
|
||||||
|
|
||||||
let contentType: string | undefined = undefined;
|
|
||||||
if (transformedHeaders["content-type"] !== undefined) {
|
|
||||||
contentType = transformedHeaders["content-type"];
|
|
||||||
}
|
|
||||||
|
|
||||||
let transformedBody: any = body;
|
|
||||||
if (ArrayBuffer.isView(body)) {
|
|
||||||
transformedBody = bufferToArrayBuffer(body);
|
|
||||||
}
|
|
||||||
|
|
||||||
const param: RequestUrlParam = {
|
|
||||||
body: transformedBody,
|
|
||||||
headers: transformedHeaders,
|
|
||||||
method: method,
|
|
||||||
url: url,
|
|
||||||
contentType: contentType,
|
|
||||||
};
|
|
||||||
|
|
||||||
const raceOfPromises = [
|
|
||||||
requestUrl(param).then((rsp) => {
|
|
||||||
const headers = rsp.headers;
|
|
||||||
const headersLower: Record<string, string> = {};
|
|
||||||
for (const key of Object.keys(headers)) {
|
|
||||||
headersLower[key.toLowerCase()] = headers[key];
|
|
||||||
}
|
|
||||||
const stream = new ReadableStream<Uint8Array>({
|
|
||||||
start(controller) {
|
|
||||||
controller.enqueue(new Uint8Array(rsp.arrayBuffer));
|
|
||||||
controller.close();
|
|
||||||
},
|
|
||||||
});
|
|
||||||
return {
|
|
||||||
response: new HttpResponse({
|
|
||||||
headers: headersLower,
|
|
||||||
statusCode: rsp.status,
|
|
||||||
body: stream,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
}),
|
|
||||||
requestTimeout(this.requestTimeoutInMs),
|
|
||||||
];
|
|
||||||
|
|
||||||
if (abortSignal) {
|
|
||||||
raceOfPromises.push(
|
|
||||||
new Promise<never>((resolve, reject) => {
|
|
||||||
abortSignal.onabort = () => {
|
|
||||||
const abortError = new Error("Request aborted");
|
|
||||||
abortError.name = "AbortError";
|
|
||||||
reject(abortError);
|
|
||||||
};
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return Promise.race(raceOfPromises);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// other stuffs
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
export const DEFAULT_S3_CONFIG: S3Config = {
|
|
||||||
s3Endpoint: "",
|
|
||||||
s3Region: "",
|
|
||||||
s3AccessKeyID: "",
|
|
||||||
s3SecretAccessKey: "",
|
|
||||||
s3BucketName: "",
|
|
||||||
bypassCorsLocally: true,
|
|
||||||
partsConcurrency: 20,
|
|
||||||
forcePathStyle: false,
|
|
||||||
remotePrefix: "",
|
|
||||||
useAccurateMTime: false, // it causes money, disable by default
|
|
||||||
reverseProxyNoSignUrl: "",
|
|
||||||
};
|
|
||||||
|
|
||||||
export type S3ObjectType = _Object;
|
|
||||||
|
|
||||||
export const simpleTransRemotePrefix = (x: string) => {
|
|
||||||
if (x === undefined) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
let y = path.posix.normalize(x.trim());
|
|
||||||
if (y === undefined || y === "" || y === "/" || y === ".") {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
if (y.startsWith("/")) {
|
|
||||||
y = y.slice(1);
|
|
||||||
}
|
|
||||||
if (!y.endsWith("/")) {
|
|
||||||
y = `${y}/`;
|
|
||||||
}
|
|
||||||
return y;
|
|
||||||
};
|
|
||||||
|
|
||||||
const getRemoteWithPrefixPath = (
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
remotePrefix: string
|
|
||||||
) => {
|
|
||||||
let key = fileOrFolderPath;
|
|
||||||
if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
|
|
||||||
// special
|
|
||||||
key = remotePrefix;
|
|
||||||
}
|
|
||||||
if (!fileOrFolderPath.startsWith("/")) {
|
|
||||||
key = `${remotePrefix}${fileOrFolderPath}`;
|
|
||||||
}
|
|
||||||
return key;
|
|
||||||
};
|
|
||||||
|
|
||||||
const getLocalNoPrefixPath = (
|
|
||||||
fileOrFolderPathWithRemotePrefix: string,
|
|
||||||
remotePrefix: string
|
|
||||||
) => {
|
|
||||||
if (
|
|
||||||
!(
|
|
||||||
fileOrFolderPathWithRemotePrefix === `${remotePrefix}` ||
|
|
||||||
fileOrFolderPathWithRemotePrefix.startsWith(`${remotePrefix}`)
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
throw Error(
|
|
||||||
`"${fileOrFolderPathWithRemotePrefix}" doesn't starts with "${remotePrefix}"`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return fileOrFolderPathWithRemotePrefix.slice(`${remotePrefix}`.length);
|
|
||||||
};
|
|
||||||
|
|
||||||
const fromS3ObjectToEntity = (
|
|
||||||
x: S3ObjectType,
|
|
||||||
remotePrefix: string,
|
|
||||||
mtimeRecords: Record<string, number>,
|
|
||||||
ctimeRecords: Record<string, number>
|
|
||||||
) => {
|
|
||||||
// console.debug(`fromS3ObjectToEntity: ${x.Key!}, ${JSON.stringify(x,null,2)}`);
|
|
||||||
// S3 officially only supports seconds precision!!!!!
|
|
||||||
const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000;
|
|
||||||
let mtimeCli = mtimeSvr;
|
|
||||||
if (x.Key! in mtimeRecords) {
|
|
||||||
const m2 = mtimeRecords[x.Key!];
|
|
||||||
if (m2 !== 0) {
|
|
||||||
// to be compatible with RClone, we read and store the time in seconds in new version!
|
|
||||||
if (m2 >= 1000000000000) {
|
|
||||||
// it's a millsecond, uploaded by old codes..
|
|
||||||
mtimeCli = m2;
|
|
||||||
} else {
|
|
||||||
// it's a second, uploaded by new codes of the plugin from March 24, 2024
|
|
||||||
mtimeCli = m2 * 1000;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const key = getLocalNoPrefixPath(x.Key!, remotePrefix);
|
|
||||||
const r: Entity = {
|
|
||||||
keyRaw: key,
|
|
||||||
mtimeSvr: mtimeSvr,
|
|
||||||
mtimeCli: mtimeCli,
|
|
||||||
sizeRaw: x.Size!,
|
|
||||||
etag: x.ETag,
|
|
||||||
synthesizedFolder: false,
|
|
||||||
};
|
|
||||||
return r;
|
|
||||||
};
|
|
||||||
|
|
||||||
const fromS3HeadObjectToEntity = (
|
|
||||||
fileOrFolderPathWithRemotePrefix: string,
|
|
||||||
x: HeadObjectCommandOutput,
|
|
||||||
remotePrefix: string
|
|
||||||
) => {
|
|
||||||
// console.debug(`fromS3HeadObjectToEntity: ${fileOrFolderPathWithRemotePrefix}: ${JSON.stringify(x,null,2)}`);
|
|
||||||
// S3 officially only supports seconds precision!!!!!
|
|
||||||
const mtimeSvr = Math.floor(x.LastModified!.valueOf() / 1000.0) * 1000;
|
|
||||||
let mtimeCli = mtimeSvr;
|
|
||||||
if (x.Metadata !== undefined) {
|
|
||||||
const m2 = Math.floor(
|
|
||||||
parseFloat(x.Metadata.mtime || x.Metadata.MTime || "0")
|
|
||||||
);
|
|
||||||
if (m2 !== 0) {
|
|
||||||
// to be compatible with RClone, we read and store the time in seconds in new version!
|
|
||||||
if (m2 >= 1000000000000) {
|
|
||||||
// it's a millsecond, uploaded by old codes..
|
|
||||||
mtimeCli = m2;
|
|
||||||
} else {
|
|
||||||
// it's a second, uploaded by new codes of the plugin from March 24, 2024
|
|
||||||
mtimeCli = m2 * 1000;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// console.debug(
|
|
||||||
// `fromS3HeadObjectToEntity, fileOrFolderPathWithRemotePrefix=${fileOrFolderPathWithRemotePrefix}, remotePrefix=${remotePrefix}, x=${JSON.stringify(
|
|
||||||
// x
|
|
||||||
// )} `
|
|
||||||
// );
|
|
||||||
const key = getLocalNoPrefixPath(
|
|
||||||
fileOrFolderPathWithRemotePrefix,
|
|
||||||
remotePrefix
|
|
||||||
);
|
|
||||||
// console.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`);
|
|
||||||
return {
|
|
||||||
keyRaw: key,
|
|
||||||
mtimeSvr: mtimeSvr,
|
|
||||||
mtimeCli: mtimeCli,
|
|
||||||
sizeRaw: x.ContentLength,
|
|
||||||
etag: x.ETag,
|
|
||||||
} as Entity;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getS3Client = (s3Config: S3Config) => {
|
|
||||||
let endpoint = s3Config.s3Endpoint;
|
|
||||||
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
|
|
||||||
endpoint = `https://${endpoint}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
let s3Client: S3Client;
|
|
||||||
if (VALID_REQURL && s3Config.bypassCorsLocally) {
|
|
||||||
s3Client = new S3Client({
|
|
||||||
region: s3Config.s3Region,
|
|
||||||
endpoint: endpoint,
|
|
||||||
forcePathStyle: s3Config.forcePathStyle,
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: s3Config.s3AccessKeyID,
|
|
||||||
secretAccessKey: s3Config.s3SecretAccessKey,
|
|
||||||
},
|
|
||||||
requestHandler: new ObsHttpHandler(
|
|
||||||
undefined,
|
|
||||||
s3Config.reverseProxyNoSignUrl
|
|
||||||
),
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
s3Client = new S3Client({
|
|
||||||
region: s3Config.s3Region,
|
|
||||||
endpoint: endpoint,
|
|
||||||
forcePathStyle: s3Config.forcePathStyle,
|
|
||||||
credentials: {
|
|
||||||
accessKeyId: s3Config.s3AccessKeyID,
|
|
||||||
secretAccessKey: s3Config.s3SecretAccessKey,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
s3Client.middlewareStack.add(
|
|
||||||
(next, context) => (args) => {
|
|
||||||
(args.request as any).headers["cache-control"] = "no-cache";
|
|
||||||
return next(args);
|
|
||||||
},
|
|
||||||
{
|
|
||||||
step: "build",
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
return s3Client;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const getRemoteMeta = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
fileOrFolderPathWithRemotePrefix: string
|
|
||||||
) => {
|
|
||||||
if (
|
|
||||||
s3Config.remotePrefix !== undefined &&
|
|
||||||
s3Config.remotePrefix !== "" &&
|
|
||||||
!fileOrFolderPathWithRemotePrefix.startsWith(s3Config.remotePrefix)
|
|
||||||
) {
|
|
||||||
throw Error(`s3 getRemoteMeta should only accept prefix-ed path`);
|
|
||||||
}
|
|
||||||
const res = await s3Client.send(
|
|
||||||
new HeadObjectCommand({
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: fileOrFolderPathWithRemotePrefix,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
return fromS3HeadObjectToEntity(
|
|
||||||
fileOrFolderPathWithRemotePrefix,
|
|
||||||
res,
|
|
||||||
s3Config.remotePrefix ?? ""
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const uploadToRemote = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault | undefined,
|
|
||||||
isRecursively: boolean,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
uploadRaw: boolean = false,
|
|
||||||
rawContent: string | ArrayBuffer = "",
|
|
||||||
rawContentMTime: number = 0,
|
|
||||||
rawContentCTime: number = 0
|
|
||||||
): Promise<UploadedType> => {
|
|
||||||
console.debug(`uploading ${fileOrFolderPath}`);
|
|
||||||
let uploadFile = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
|
||||||
throw Error(
|
|
||||||
`uploadToRemote(s3) you have password but remoteEncryptedKey is empty!`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
uploadFile = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
uploadFile = getRemoteWithPrefixPath(uploadFile, s3Config.remotePrefix ?? "");
|
|
||||||
// console.debug(`actual uploadFile=${uploadFile}`);
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (isFolder && isRecursively) {
|
|
||||||
throw Error("upload function doesn't implement recursive function yet!");
|
|
||||||
} else if (isFolder && !isRecursively) {
|
|
||||||
if (uploadRaw) {
|
|
||||||
throw Error(`you specify uploadRaw, but you also provide a folder key!`);
|
|
||||||
}
|
|
||||||
// folder
|
|
||||||
let mtime = 0;
|
|
||||||
let ctime = 0;
|
|
||||||
const s = await vault?.adapter?.stat(fileOrFolderPath);
|
|
||||||
if (s !== undefined && s !== null) {
|
|
||||||
mtime = s.mtime;
|
|
||||||
ctime = s.ctime;
|
|
||||||
}
|
|
||||||
const contentType = DEFAULT_CONTENT_TYPE;
|
|
||||||
await s3Client.send(
|
|
||||||
new PutObjectCommand({
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: uploadFile,
|
|
||||||
Body: "",
|
|
||||||
ContentType: contentType,
|
|
||||||
Metadata: {
|
|
||||||
MTime: `${mtime / 1000.0}`,
|
|
||||||
CTime: `${ctime / 1000.0}`,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
);
|
|
||||||
const res = await getRemoteMeta(s3Client, s3Config, uploadFile);
|
|
||||||
return {
|
|
||||||
entity: res,
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
// file
|
|
||||||
// we ignore isRecursively parameter here
|
|
||||||
let contentType = DEFAULT_CONTENT_TYPE;
|
|
||||||
if (cipher.isPasswordEmpty()) {
|
|
||||||
contentType =
|
|
||||||
mime.contentType(
|
|
||||||
mime.lookup(fileOrFolderPath) || DEFAULT_CONTENT_TYPE
|
|
||||||
) || DEFAULT_CONTENT_TYPE;
|
|
||||||
}
|
|
||||||
let localContent = undefined;
|
|
||||||
let mtime = 0;
|
|
||||||
let ctime = 0;
|
|
||||||
if (uploadRaw) {
|
|
||||||
if (typeof rawContent === "string") {
|
|
||||||
localContent = new TextEncoder().encode(rawContent).buffer;
|
|
||||||
} else {
|
|
||||||
localContent = rawContent;
|
|
||||||
}
|
|
||||||
mtime = rawContentMTime;
|
|
||||||
ctime = rawContentCTime;
|
|
||||||
} else {
|
|
||||||
if (vault === undefined) {
|
|
||||||
throw new Error(
|
|
||||||
`the vault variable is not passed but we want to read ${fileOrFolderPath} for S3`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
localContent = await vault.adapter.readBinary(fileOrFolderPath);
|
|
||||||
const s = await vault.adapter.stat(fileOrFolderPath);
|
|
||||||
if (s !== undefined && s !== null) {
|
|
||||||
mtime = s.mtime;
|
|
||||||
ctime = s.ctime;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let remoteContent = localContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteContent = await cipher.encryptContent(localContent);
|
|
||||||
}
|
|
||||||
|
|
||||||
const bytesIn5MB = 5242880;
|
|
||||||
const body = new Uint8Array(remoteContent);
|
|
||||||
|
|
||||||
const upload = new Upload({
|
|
||||||
client: s3Client,
|
|
||||||
queueSize: s3Config.partsConcurrency, // concurrency
|
|
||||||
partSize: bytesIn5MB, // minimal 5MB by default
|
|
||||||
leavePartsOnError: false,
|
|
||||||
params: {
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: uploadFile,
|
|
||||||
Body: body,
|
|
||||||
ContentType: contentType,
|
|
||||||
Metadata: {
|
|
||||||
MTime: `${mtime / 1000.0}`,
|
|
||||||
CTime: `${ctime / 1000.0}`,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
upload.on("httpUploadProgress", (progress) => {
|
|
||||||
// console.info(progress);
|
|
||||||
});
|
|
||||||
await upload.done();
|
|
||||||
|
|
||||||
const res = await getRemoteMeta(s3Client, s3Config, uploadFile);
|
|
||||||
// console.debug(
|
|
||||||
// `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}`
|
|
||||||
// );
|
|
||||||
return {
|
|
||||||
entity: res,
|
|
||||||
mtimeCli: mtime,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const listFromRemoteRaw = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
prefixOfRawKeys?: string
|
|
||||||
) => {
|
|
||||||
const confCmd = {
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
} as ListObjectsV2CommandInput;
|
|
||||||
if (prefixOfRawKeys !== undefined && prefixOfRawKeys !== "") {
|
|
||||||
confCmd.Prefix = prefixOfRawKeys;
|
|
||||||
}
|
|
||||||
|
|
||||||
const contents = [] as _Object[];
|
|
||||||
const mtimeRecords: Record<string, number> = {};
|
|
||||||
const ctimeRecords: Record<string, number> = {};
|
|
||||||
const queueHead = new PQueue({
|
|
||||||
concurrency: s3Config.partsConcurrency,
|
|
||||||
autoStart: true,
|
|
||||||
});
|
|
||||||
queueHead.on("error", (error) => {
|
|
||||||
queueHead.pause();
|
|
||||||
queueHead.clear();
|
|
||||||
throw error;
|
|
||||||
});
|
|
||||||
|
|
||||||
let isTruncated = true;
|
|
||||||
do {
|
|
||||||
const rsp = await s3Client.send(new ListObjectsV2Command(confCmd));
|
|
||||||
|
|
||||||
if (rsp.$metadata.httpStatusCode !== 200) {
|
|
||||||
throw Error("some thing bad while listing remote!");
|
|
||||||
}
|
|
||||||
if (rsp.Contents === undefined) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
contents.push(...rsp.Contents);
|
|
||||||
|
|
||||||
if (s3Config.useAccurateMTime) {
|
|
||||||
// head requests of all objects, love it
|
|
||||||
for (const content of rsp.Contents) {
|
|
||||||
queueHead.add(async () => {
|
|
||||||
const rspHead = await s3Client.send(
|
|
||||||
new HeadObjectCommand({
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: content.Key,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
if (rspHead.$metadata.httpStatusCode !== 200) {
|
|
||||||
throw Error("some thing bad while heading single object!");
|
|
||||||
}
|
|
||||||
if (rspHead.Metadata === undefined) {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
mtimeRecords[content.Key!] = Math.floor(
|
|
||||||
parseFloat(
|
|
||||||
rspHead.Metadata.mtime || rspHead.Metadata.MTime || "0"
|
|
||||||
)
|
|
||||||
);
|
|
||||||
ctimeRecords[content.Key!] = Math.floor(
|
|
||||||
parseFloat(
|
|
||||||
rspHead.Metadata.ctime || rspHead.Metadata.CTime || "0"
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
isTruncated = rsp.IsTruncated ?? false;
|
|
||||||
confCmd.ContinuationToken = rsp.NextContinuationToken;
|
|
||||||
if (
|
|
||||||
isTruncated &&
|
|
||||||
(confCmd.ContinuationToken === undefined ||
|
|
||||||
confCmd.ContinuationToken === "")
|
|
||||||
) {
|
|
||||||
throw Error("isTruncated is true but no continuationToken provided");
|
|
||||||
}
|
|
||||||
} while (isTruncated);
|
|
||||||
|
|
||||||
// wait for any head requests
|
|
||||||
await queueHead.onIdle();
|
|
||||||
|
|
||||||
// ensemble fake rsp
|
|
||||||
// in the end, we need to transform the response list
|
|
||||||
// back to the local contents-alike list
|
|
||||||
return contents.map((x) =>
|
|
||||||
fromS3ObjectToEntity(
|
|
||||||
x,
|
|
||||||
s3Config.remotePrefix ?? "",
|
|
||||||
mtimeRecords,
|
|
||||||
ctimeRecords
|
|
||||||
)
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listAllFromRemote = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config
|
|
||||||
) => {
|
|
||||||
const res = (
|
|
||||||
await listFromRemoteRaw(s3Client, s3Config, s3Config.remotePrefix)
|
|
||||||
).filter((x) => x.keyRaw !== "" && x.keyRaw !== "/");
|
|
||||||
return res;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The Body of resp of aws GetObject has mix types
|
|
||||||
* and we want to get ArrayBuffer here.
|
|
||||||
* See https://github.com/aws/aws-sdk-js-v3/issues/1877
|
|
||||||
* @param b The Body of GetObject
|
|
||||||
* @returns Promise<ArrayBuffer>
|
|
||||||
*/
|
|
||||||
const getObjectBodyToArrayBuffer = async (
|
|
||||||
b: Readable | ReadableStream | Blob | undefined
|
|
||||||
) => {
|
|
||||||
if (b === undefined) {
|
|
||||||
throw Error(`ObjectBody is undefined and don't know how to deal with it`);
|
|
||||||
}
|
|
||||||
if (b instanceof Readable) {
|
|
||||||
return (await new Promise((resolve, reject) => {
|
|
||||||
const chunks: Uint8Array[] = [];
|
|
||||||
b.on("data", (chunk) => chunks.push(chunk));
|
|
||||||
b.on("error", reject);
|
|
||||||
b.on("end", () => resolve(bufferToArrayBuffer(Buffer.concat(chunks))));
|
|
||||||
})) as ArrayBuffer;
|
|
||||||
} else if (b instanceof ReadableStream) {
|
|
||||||
return await new Response(b, {}).arrayBuffer();
|
|
||||||
} else if (b instanceof Blob) {
|
|
||||||
return await b.arrayBuffer();
|
|
||||||
} else {
|
|
||||||
throw TypeError(`The type of ${b} is not one of the supported types`);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const downloadFromRemoteRaw = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
fileOrFolderPathWithRemotePrefix: string
|
|
||||||
) => {
|
|
||||||
if (
|
|
||||||
s3Config.remotePrefix !== undefined &&
|
|
||||||
s3Config.remotePrefix !== "" &&
|
|
||||||
!fileOrFolderPathWithRemotePrefix.startsWith(s3Config.remotePrefix)
|
|
||||||
) {
|
|
||||||
throw Error(`downloadFromRemoteRaw should only accept prefix-ed path`);
|
|
||||||
}
|
|
||||||
const data = await s3Client.send(
|
|
||||||
new GetObjectCommand({
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: fileOrFolderPathWithRemotePrefix,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
const bodyContents = await getObjectBodyToArrayBuffer(data.Body);
|
|
||||||
return bodyContents;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const downloadFromRemote = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault,
|
|
||||||
mtime: number,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string,
|
|
||||||
skipSaving: boolean = false
|
|
||||||
) => {
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (!skipSaving) {
|
|
||||||
await mkdirpInVault(fileOrFolderPath, vault);
|
|
||||||
}
|
|
||||||
|
|
||||||
// the file is always local file
|
|
||||||
// we need to encrypt it
|
|
||||||
|
|
||||||
if (isFolder) {
|
|
||||||
// mkdirp locally is enough
|
|
||||||
// do nothing here
|
|
||||||
return new ArrayBuffer(0);
|
|
||||||
} else {
|
|
||||||
let downloadFile = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
downloadFile = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
downloadFile = getRemoteWithPrefixPath(
|
|
||||||
downloadFile,
|
|
||||||
s3Config.remotePrefix ?? ""
|
|
||||||
);
|
|
||||||
const remoteContent = await downloadFromRemoteRaw(
|
|
||||||
s3Client,
|
|
||||||
s3Config,
|
|
||||||
downloadFile
|
|
||||||
);
|
|
||||||
let localContent = remoteContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
localContent = await cipher.decryptContent(remoteContent);
|
|
||||||
}
|
|
||||||
if (!skipSaving) {
|
|
||||||
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
|
|
||||||
mtime: mtime,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return localContent;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This function deals with file normally and "folder" recursively.
|
|
||||||
* @param s3Client
|
|
||||||
* @param s3Config
|
|
||||||
* @param fileOrFolderPath
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
export const deleteFromRemote = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
synthesizedFolder: boolean = false
|
|
||||||
) => {
|
|
||||||
if (fileOrFolderPath === "/") {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (synthesizedFolder) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let remoteFileName = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteFileName = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
remoteFileName = getRemoteWithPrefixPath(
|
|
||||||
remoteFileName,
|
|
||||||
s3Config.remotePrefix ?? ""
|
|
||||||
);
|
|
||||||
await s3Client.send(
|
|
||||||
new DeleteObjectCommand({
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: remoteFileName,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
|
|
||||||
if (fileOrFolderPath.endsWith("/") && cipher.isPasswordEmpty()) {
|
|
||||||
const x = await listFromRemoteRaw(s3Client, s3Config, remoteFileName);
|
|
||||||
x.forEach(async (element) => {
|
|
||||||
await s3Client.send(
|
|
||||||
new DeleteObjectCommand({
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
Key: element.key,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
});
|
|
||||||
} else if (fileOrFolderPath.endsWith("/") && !cipher.isPasswordEmpty()) {
|
|
||||||
// TODO
|
|
||||||
} else {
|
|
||||||
// pass
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check the config of S3 by heading bucket
|
|
||||||
* https://stackoverflow.com/questions/50842835
|
|
||||||
*
|
|
||||||
* Updated on 20240102:
|
|
||||||
* Users are not always have permission of heading bucket,
|
|
||||||
* so we need to use listing objects instead...
|
|
||||||
*
|
|
||||||
* @param s3Client
|
|
||||||
* @param s3Config
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
export const checkConnectivity = async (
|
|
||||||
s3Client: S3Client,
|
|
||||||
s3Config: S3Config,
|
|
||||||
callbackFunc?: any
|
|
||||||
) => {
|
|
||||||
try {
|
|
||||||
// TODO: no universal way now, just check this in connectivity
|
|
||||||
if (Platform.isIosApp && s3Config.s3Endpoint.startsWith("http://")) {
|
|
||||||
throw Error(
|
|
||||||
`Your s3 endpoint could only be https, not http, because of the iOS restriction.`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// const results = await s3Client.send(
|
|
||||||
// new HeadBucketCommand({ Bucket: s3Config.s3BucketName })
|
|
||||||
// );
|
|
||||||
// very simplified version of listing objects
|
|
||||||
const confCmd = {
|
|
||||||
Bucket: s3Config.s3BucketName,
|
|
||||||
} as ListObjectsV2CommandInput;
|
|
||||||
const results = await s3Client.send(new ListObjectsV2Command(confCmd));
|
|
||||||
|
|
||||||
if (
|
|
||||||
results === undefined ||
|
|
||||||
results.$metadata === undefined ||
|
|
||||||
results.$metadata.httpStatusCode === undefined
|
|
||||||
) {
|
|
||||||
const err = "results or $metadata or httStatusCode is undefined";
|
|
||||||
console.debug(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return results.$metadata.httpStatusCode === 200;
|
|
||||||
} catch (err: any) {
|
|
||||||
console.debug(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
if (s3Config.s3Endpoint.contains(s3Config.s3BucketName)) {
|
|
||||||
const err2 = new AggregateError([
|
|
||||||
err,
|
|
||||||
new Error(
|
|
||||||
"Maybe you've included the bucket name inside the endpoint setting. Please remove the bucket name and try again."
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
callbackFunc(err2);
|
|
||||||
} else {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
|
@ -1,606 +0,0 @@
|
||||||
import { Buffer } from "buffer";
|
|
||||||
import { Platform, Vault, requestUrl } from "obsidian";
|
|
||||||
|
|
||||||
import { Queue } from "@fyears/tsqueue";
|
|
||||||
import chunk from "lodash/chunk";
|
|
||||||
import flatten from "lodash/flatten";
|
|
||||||
import cloneDeep from "lodash/cloneDeep";
|
|
||||||
import { getReasonPhrase } from "http-status-codes";
|
|
||||||
import { Entity, UploadedType, VALID_REQURL, WebdavConfig } from "./baseTypes";
|
|
||||||
import { bufferToArrayBuffer, getPathFolder, mkdirpInVault } from "./misc";
|
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
|
|
||||||
import type {
|
|
||||||
FileStat,
|
|
||||||
WebDAVClient,
|
|
||||||
RequestOptionsWithState,
|
|
||||||
// Response,
|
|
||||||
// ResponseDataDetailed,
|
|
||||||
} from "webdav";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* https://stackoverflow.com/questions/32850898/how-to-check-if-a-string-has-any-non-iso-8859-1-characters-with-javascript
|
|
||||||
* @param str
|
|
||||||
* @returns true if all are iso 8859 1 chars
|
|
||||||
*/
|
|
||||||
function onlyAscii(str: string) {
|
|
||||||
return !/[^\u0000-\u00ff]/g.test(str);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* https://stackoverflow.com/questions/12539574/
|
|
||||||
* @param obj
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
function objKeyToLower(obj: Record<string, string>) {
|
|
||||||
return Object.fromEntries(
|
|
||||||
Object.entries(obj).map(([k, v]) => [k.toLowerCase(), v])
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// @ts-ignore
|
|
||||||
import { getPatcher } from "webdav/dist/web/index.js";
|
|
||||||
if (VALID_REQURL) {
|
|
||||||
getPatcher().patch(
|
|
||||||
"request",
|
|
||||||
async (options: RequestOptionsWithState): Promise<Response> => {
|
|
||||||
const transformedHeaders = objKeyToLower({ ...options.headers });
|
|
||||||
delete transformedHeaders["host"];
|
|
||||||
delete transformedHeaders["content-length"];
|
|
||||||
|
|
||||||
const reqContentType =
|
|
||||||
transformedHeaders["accept"] ?? transformedHeaders["content-type"];
|
|
||||||
|
|
||||||
const retractedHeaders = { ...transformedHeaders };
|
|
||||||
if (retractedHeaders.hasOwnProperty("authorization")) {
|
|
||||||
retractedHeaders["authorization"] = "<retracted>";
|
|
||||||
}
|
|
||||||
|
|
||||||
console.debug(`before request:`);
|
|
||||||
console.debug(`url: ${options.url}`);
|
|
||||||
console.debug(`method: ${options.method}`);
|
|
||||||
console.debug(`headers: ${JSON.stringify(retractedHeaders, null, 2)}`);
|
|
||||||
console.debug(`reqContentType: ${reqContentType}`);
|
|
||||||
|
|
||||||
let r = await requestUrl({
|
|
||||||
url: options.url,
|
|
||||||
method: options.method,
|
|
||||||
body: options.data as string | ArrayBuffer,
|
|
||||||
headers: transformedHeaders,
|
|
||||||
contentType: reqContentType,
|
|
||||||
throw: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (
|
|
||||||
r.status === 401 &&
|
|
||||||
Platform.isIosApp &&
|
|
||||||
!options.url.endsWith("/") &&
|
|
||||||
!options.url.endsWith(".md") &&
|
|
||||||
options.method.toUpperCase() === "PROPFIND"
|
|
||||||
) {
|
|
||||||
// don't ask me why,
|
|
||||||
// some webdav servers have some mysterious behaviours,
|
|
||||||
// if a folder doesn't exist without slash, the servers return 401 instead of 404
|
|
||||||
// here is a dirty hack that works
|
|
||||||
console.debug(`so we have 401, try appending request url with slash`);
|
|
||||||
r = await requestUrl({
|
|
||||||
url: `${options.url}/`,
|
|
||||||
method: options.method,
|
|
||||||
body: options.data as string | ArrayBuffer,
|
|
||||||
headers: transformedHeaders,
|
|
||||||
contentType: reqContentType,
|
|
||||||
throw: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
console.debug(`after request:`);
|
|
||||||
const rspHeaders = objKeyToLower({ ...r.headers });
|
|
||||||
console.debug(`rspHeaders: ${JSON.stringify(rspHeaders, null, 2)}`);
|
|
||||||
for (let key in rspHeaders) {
|
|
||||||
if (rspHeaders.hasOwnProperty(key)) {
|
|
||||||
// avoid the error:
|
|
||||||
// Failed to read the 'headers' property from 'ResponseInit': String contains non ISO-8859-1 code point.
|
|
||||||
// const possibleNonAscii = [
|
|
||||||
// "Content-Disposition",
|
|
||||||
// "X-Accel-Redirect",
|
|
||||||
// "X-Outfilename",
|
|
||||||
// "X-Sendfile"
|
|
||||||
// ];
|
|
||||||
// for (const p of possibleNonAscii) {
|
|
||||||
// if (key === p || key === p.toLowerCase()) {
|
|
||||||
// rspHeaders[key] = encodeURIComponent(rspHeaders[key]);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
if (!onlyAscii(rspHeaders[key])) {
|
|
||||||
console.debug(`rspHeaders[key] needs encode: ${key}`);
|
|
||||||
rspHeaders[key] = encodeURIComponent(rspHeaders[key]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let r2: Response | undefined = undefined;
|
|
||||||
const statusText = getReasonPhrase(r.status);
|
|
||||||
console.debug(`statusText: ${statusText}`);
|
|
||||||
if ([101, 103, 204, 205, 304].includes(r.status)) {
|
|
||||||
// A null body status is a status that is 101, 103, 204, 205, or 304.
|
|
||||||
// https://fetch.spec.whatwg.org/#statuses
|
|
||||||
// fix this: Failed to construct 'Response': Response with null body status cannot have body
|
|
||||||
r2 = new Response(null, {
|
|
||||||
status: r.status,
|
|
||||||
statusText: statusText,
|
|
||||||
headers: rspHeaders,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
r2 = new Response(r.arrayBuffer, {
|
|
||||||
status: r.status,
|
|
||||||
statusText: statusText,
|
|
||||||
headers: rspHeaders,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return r2;
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// @ts-ignore
|
|
||||||
import { AuthType, BufferLike, createClient } from "webdav/dist/web/index.js";
|
|
||||||
export type { WebDAVClient } from "webdav";
|
|
||||||
|
|
||||||
export const DEFAULT_WEBDAV_CONFIG = {
|
|
||||||
address: "",
|
|
||||||
username: "",
|
|
||||||
password: "",
|
|
||||||
authType: "basic",
|
|
||||||
manualRecursive: true,
|
|
||||||
depth: "manual_1",
|
|
||||||
remoteBaseDir: "",
|
|
||||||
} as WebdavConfig;
|
|
||||||
|
|
||||||
const getWebdavPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
|
||||||
let key = fileOrFolderPath;
|
|
||||||
if (fileOrFolderPath === "/" || fileOrFolderPath === "") {
|
|
||||||
// special
|
|
||||||
key = `/${remoteBaseDir}/`;
|
|
||||||
} else if (fileOrFolderPath.startsWith("/")) {
|
|
||||||
console.warn(
|
|
||||||
`why the path ${fileOrFolderPath} starts with '/'? but we just go on.`
|
|
||||||
);
|
|
||||||
key = `/${remoteBaseDir}${fileOrFolderPath}`;
|
|
||||||
} else {
|
|
||||||
key = `/${remoteBaseDir}/${fileOrFolderPath}`;
|
|
||||||
}
|
|
||||||
return key;
|
|
||||||
};
|
|
||||||
|
|
||||||
const getNormPath = (fileOrFolderPath: string, remoteBaseDir: string) => {
|
|
||||||
if (
|
|
||||||
!(
|
|
||||||
fileOrFolderPath === `/${remoteBaseDir}` ||
|
|
||||||
fileOrFolderPath.startsWith(`/${remoteBaseDir}/`)
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
throw Error(
|
|
||||||
`"${fileOrFolderPath}" doesn't starts with "/${remoteBaseDir}/"`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return fileOrFolderPath.slice(`/${remoteBaseDir}/`.length);
|
|
||||||
};
|
|
||||||
|
|
||||||
const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => {
|
|
||||||
let key = getNormPath(x.filename, remoteBaseDir);
|
|
||||||
if (x.type === "directory" && !key.endsWith("/")) {
|
|
||||||
key = `${key}/`;
|
|
||||||
}
|
|
||||||
const mtimeSvr = Date.parse(x.lastmod).valueOf();
|
|
||||||
return {
|
|
||||||
keyRaw: key,
|
|
||||||
mtimeSvr: mtimeSvr,
|
|
||||||
mtimeCli: mtimeSvr, // no universal way to set mtime in webdav
|
|
||||||
sizeRaw: x.size,
|
|
||||||
etag: x.etag,
|
|
||||||
} as Entity;
|
|
||||||
};
|
|
||||||
|
|
||||||
export class WrappedWebdavClient {
|
|
||||||
webdavConfig: WebdavConfig;
|
|
||||||
remoteBaseDir: string;
|
|
||||||
client!: WebDAVClient;
|
|
||||||
vaultFolderExists: boolean;
|
|
||||||
saveUpdatedConfigFunc: () => Promise<any>;
|
|
||||||
constructor(
|
|
||||||
webdavConfig: WebdavConfig,
|
|
||||||
remoteBaseDir: string,
|
|
||||||
saveUpdatedConfigFunc: () => Promise<any>
|
|
||||||
) {
|
|
||||||
this.webdavConfig = cloneDeep(webdavConfig);
|
|
||||||
this.webdavConfig.address = encodeURI(this.webdavConfig.address);
|
|
||||||
this.remoteBaseDir = remoteBaseDir;
|
|
||||||
this.vaultFolderExists = false;
|
|
||||||
this.saveUpdatedConfigFunc = saveUpdatedConfigFunc;
|
|
||||||
}
|
|
||||||
|
|
||||||
init = async () => {
|
|
||||||
// init client if not inited
|
|
||||||
if (this.client !== undefined) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Platform.isIosApp && !this.webdavConfig.address.startsWith("https")) {
|
|
||||||
throw Error(
|
|
||||||
`Your webdav address could only be https, not http, because of the iOS restriction.`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const headers = {
|
|
||||||
"Cache-Control": "no-cache",
|
|
||||||
};
|
|
||||||
if (
|
|
||||||
this.webdavConfig.username !== "" &&
|
|
||||||
this.webdavConfig.password !== ""
|
|
||||||
) {
|
|
||||||
this.client = createClient(this.webdavConfig.address, {
|
|
||||||
username: this.webdavConfig.username,
|
|
||||||
password: this.webdavConfig.password,
|
|
||||||
headers: headers,
|
|
||||||
authType:
|
|
||||||
this.webdavConfig.authType === "digest"
|
|
||||||
? AuthType.Digest
|
|
||||||
: AuthType.Password,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
console.info("no password");
|
|
||||||
this.client = createClient(this.webdavConfig.address, {
|
|
||||||
headers: headers,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// check vault folder
|
|
||||||
if (this.vaultFolderExists) {
|
|
||||||
// pass
|
|
||||||
} else {
|
|
||||||
const res = await this.client.exists(`/${this.remoteBaseDir}/`);
|
|
||||||
if (res) {
|
|
||||||
// console.info("remote vault folder exits!");
|
|
||||||
this.vaultFolderExists = true;
|
|
||||||
} else {
|
|
||||||
console.info("remote vault folder not exists, creating");
|
|
||||||
await this.client.createDirectory(`/${this.remoteBaseDir}/`);
|
|
||||||
console.info("remote vault folder created!");
|
|
||||||
this.vaultFolderExists = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// adjust depth parameter
|
|
||||||
if (
|
|
||||||
this.webdavConfig.depth === "auto" ||
|
|
||||||
this.webdavConfig.depth === "auto_1" ||
|
|
||||||
this.webdavConfig.depth === "auto_infinity" ||
|
|
||||||
this.webdavConfig.depth === "auto_unknown"
|
|
||||||
) {
|
|
||||||
this.webdavConfig.depth = "manual_1";
|
|
||||||
this.webdavConfig.manualRecursive = true;
|
|
||||||
if (this.saveUpdatedConfigFunc !== undefined) {
|
|
||||||
await this.saveUpdatedConfigFunc();
|
|
||||||
console.info(
|
|
||||||
`webdav depth="auto_???" is changed to ${this.webdavConfig.depth}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export const getWebdavClient = (
|
|
||||||
webdavConfig: WebdavConfig,
|
|
||||||
remoteBaseDir: string,
|
|
||||||
saveUpdatedConfigFunc: () => Promise<any>
|
|
||||||
) => {
|
|
||||||
return new WrappedWebdavClient(
|
|
||||||
webdavConfig,
|
|
||||||
remoteBaseDir,
|
|
||||||
saveUpdatedConfigFunc
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param client
|
|
||||||
* @param remotePath It should be prefix-ed already
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
export const getRemoteMeta = async (
|
|
||||||
client: WrappedWebdavClient,
|
|
||||||
remotePath: string
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
console.debug(`getRemoteMeta remotePath = ${remotePath}`);
|
|
||||||
const res = (await client.client.stat(remotePath, {
|
|
||||||
details: false,
|
|
||||||
})) as FileStat;
|
|
||||||
console.debug(`getRemoteMeta res=${JSON.stringify(res)}`);
|
|
||||||
return fromWebdavItemToEntity(res, client.remoteBaseDir);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const uploadToRemote = async (
|
|
||||||
client: WrappedWebdavClient,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault | undefined,
|
|
||||||
isRecursively: boolean,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
uploadRaw: boolean = false,
|
|
||||||
rawContent: string | ArrayBuffer = ""
|
|
||||||
): Promise<UploadedType> => {
|
|
||||||
await client.init();
|
|
||||||
let uploadFile = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
|
||||||
throw Error(
|
|
||||||
`uploadToRemote(webdav) you have password but remoteEncryptedKey is empty!`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
uploadFile = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
uploadFile = getWebdavPath(uploadFile, client.remoteBaseDir);
|
|
||||||
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (isFolder && isRecursively) {
|
|
||||||
throw Error("upload function doesn't implement recursive function yet!");
|
|
||||||
} else if (isFolder && !isRecursively) {
|
|
||||||
if (uploadRaw) {
|
|
||||||
throw Error(`you specify uploadRaw, but you also provide a folder key!`);
|
|
||||||
}
|
|
||||||
// folder
|
|
||||||
if (cipher.isPasswordEmpty() || cipher.isFolderAware()) {
|
|
||||||
// if not encrypted, || encrypted isFolderAware, mkdir a remote folder
|
|
||||||
await client.client.createDirectory(uploadFile, {
|
|
||||||
recursive: true,
|
|
||||||
});
|
|
||||||
const res = await getRemoteMeta(client, uploadFile);
|
|
||||||
return {
|
|
||||||
entity: res,
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
// if encrypted && !isFolderAware(),
|
|
||||||
// upload a fake file with the encrypted file name
|
|
||||||
await client.client.putFileContents(uploadFile, "", {
|
|
||||||
overwrite: true,
|
|
||||||
onUploadProgress: (progress: any) => {
|
|
||||||
// console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
entity: await getRemoteMeta(client, uploadFile),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// file
|
|
||||||
// we ignore isRecursively parameter here
|
|
||||||
let localContent: ArrayBuffer | undefined = undefined;
|
|
||||||
let mtimeCli: number | undefined = undefined;
|
|
||||||
if (uploadRaw) {
|
|
||||||
if (typeof rawContent === "string") {
|
|
||||||
localContent = new TextEncoder().encode(rawContent).buffer;
|
|
||||||
} else {
|
|
||||||
localContent = rawContent;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (vault == undefined) {
|
|
||||||
throw new Error(
|
|
||||||
`the vault variable is not passed but we want to read ${fileOrFolderPath} for webdav`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
localContent = await vault.adapter.readBinary(fileOrFolderPath);
|
|
||||||
mtimeCli = (await vault.adapter.stat(fileOrFolderPath))?.mtime;
|
|
||||||
}
|
|
||||||
let remoteContent = localContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteContent = await cipher.encryptContent(localContent);
|
|
||||||
}
|
|
||||||
// updated 20220326: the algorithm guarantee this
|
|
||||||
// // we need to create folders before uploading
|
|
||||||
// const dir = getPathFolder(uploadFile);
|
|
||||||
// if (dir !== "/" && dir !== "") {
|
|
||||||
// await client.client.createDirectory(dir, { recursive: true });
|
|
||||||
// }
|
|
||||||
await client.client.putFileContents(uploadFile, remoteContent, {
|
|
||||||
overwrite: true,
|
|
||||||
onUploadProgress: (progress: any) => {
|
|
||||||
console.info(`Uploaded ${progress.loaded} bytes of ${progress.total}`);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
entity: await getRemoteMeta(client, uploadFile),
|
|
||||||
mtimeCli: mtimeCli,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const listAllFromRemote = async (client: WrappedWebdavClient) => {
|
|
||||||
await client.init();
|
|
||||||
|
|
||||||
let contents = [] as FileStat[];
|
|
||||||
if (
|
|
||||||
client.webdavConfig.depth === "auto" ||
|
|
||||||
client.webdavConfig.depth === "auto_unknown" ||
|
|
||||||
client.webdavConfig.depth === "auto_1" ||
|
|
||||||
client.webdavConfig.depth === "auto_infinity" /* don't trust auto now */ ||
|
|
||||||
client.webdavConfig.depth === "manual_1"
|
|
||||||
) {
|
|
||||||
// the remote doesn't support infinity propfind,
|
|
||||||
// we need to do a bfs here
|
|
||||||
const q = new Queue([`/${client.remoteBaseDir}`]);
|
|
||||||
const CHUNK_SIZE = 10;
|
|
||||||
while (q.length > 0) {
|
|
||||||
const itemsToFetch: string[] = [];
|
|
||||||
while (q.length > 0) {
|
|
||||||
itemsToFetch.push(q.pop()!);
|
|
||||||
}
|
|
||||||
const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE);
|
|
||||||
// console.debug(itemsToFetchChunks);
|
|
||||||
const subContents = [] as FileStat[];
|
|
||||||
for (const singleChunk of itemsToFetchChunks) {
|
|
||||||
const r = singleChunk.map((x) => {
|
|
||||||
return client.client.getDirectoryContents(x, {
|
|
||||||
deep: false,
|
|
||||||
details: false /* no need for verbose details here */,
|
|
||||||
// TODO: to support .obsidian,
|
|
||||||
// we need to load all files including dot,
|
|
||||||
// anyway to reduce the resources?
|
|
||||||
// glob: "/**" /* avoid dot files by using glob */,
|
|
||||||
}) as Promise<FileStat[]>;
|
|
||||||
});
|
|
||||||
const r2 = flatten(await Promise.all(r));
|
|
||||||
subContents.push(...r2);
|
|
||||||
}
|
|
||||||
for (let i = 0; i < subContents.length; ++i) {
|
|
||||||
const f = subContents[i];
|
|
||||||
contents.push(f);
|
|
||||||
if (f.type === "directory") {
|
|
||||||
q.push(f.filename);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// the remote supports infinity propfind
|
|
||||||
contents = (await client.client.getDirectoryContents(
|
|
||||||
`/${client.remoteBaseDir}`,
|
|
||||||
{
|
|
||||||
deep: true,
|
|
||||||
details: false /* no need for verbose details here */,
|
|
||||||
// TODO: to support .obsidian,
|
|
||||||
// we need to load all files including dot,
|
|
||||||
// anyway to reduce the resources?
|
|
||||||
// glob: "/**" /* avoid dot files by using glob */,
|
|
||||||
}
|
|
||||||
)) as FileStat[];
|
|
||||||
}
|
|
||||||
return contents.map((x) => fromWebdavItemToEntity(x, client.remoteBaseDir));
|
|
||||||
};
|
|
||||||
|
|
||||||
const downloadFromRemoteRaw = async (
|
|
||||||
client: WrappedWebdavClient,
|
|
||||||
remotePath: string
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
// console.info(`getWebdavPath=${remotePath}`);
|
|
||||||
const buff = (await client.client.getFileContents(remotePath)) as BufferLike;
|
|
||||||
if (buff instanceof ArrayBuffer) {
|
|
||||||
return buff;
|
|
||||||
} else if (buff instanceof Buffer) {
|
|
||||||
return bufferToArrayBuffer(buff);
|
|
||||||
}
|
|
||||||
throw Error(`unexpected file content result with type ${typeof buff}`);
|
|
||||||
};
|
|
||||||
|
|
||||||
export const downloadFromRemote = async (
|
|
||||||
client: WrappedWebdavClient,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
vault: Vault,
|
|
||||||
mtime: number,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = "",
|
|
||||||
skipSaving: boolean = false
|
|
||||||
) => {
|
|
||||||
await client.init();
|
|
||||||
|
|
||||||
const isFolder = fileOrFolderPath.endsWith("/");
|
|
||||||
|
|
||||||
if (!skipSaving) {
|
|
||||||
await mkdirpInVault(fileOrFolderPath, vault);
|
|
||||||
}
|
|
||||||
|
|
||||||
// the file is always local file
|
|
||||||
// we need to encrypt it
|
|
||||||
|
|
||||||
if (isFolder) {
|
|
||||||
// mkdirp locally is enough
|
|
||||||
// do nothing here
|
|
||||||
return new ArrayBuffer(0);
|
|
||||||
} else {
|
|
||||||
let downloadFile = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
downloadFile = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
downloadFile = getWebdavPath(downloadFile, client.remoteBaseDir);
|
|
||||||
// console.info(`downloadFile=${downloadFile}`);
|
|
||||||
const remoteContent = await downloadFromRemoteRaw(client, downloadFile);
|
|
||||||
let localContent = remoteContent;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
localContent = await cipher.decryptContent(remoteContent);
|
|
||||||
}
|
|
||||||
if (!skipSaving) {
|
|
||||||
await vault.adapter.writeBinary(fileOrFolderPath, localContent, {
|
|
||||||
mtime: mtime,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return localContent;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const deleteFromRemote = async (
|
|
||||||
client: WrappedWebdavClient,
|
|
||||||
fileOrFolderPath: string,
|
|
||||||
cipher: Cipher,
|
|
||||||
remoteEncryptedKey: string = ""
|
|
||||||
) => {
|
|
||||||
if (fileOrFolderPath === "/") {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let remoteFileName = fileOrFolderPath;
|
|
||||||
if (!cipher.isPasswordEmpty()) {
|
|
||||||
remoteFileName = remoteEncryptedKey;
|
|
||||||
}
|
|
||||||
remoteFileName = getWebdavPath(remoteFileName, client.remoteBaseDir);
|
|
||||||
|
|
||||||
await client.init();
|
|
||||||
try {
|
|
||||||
await client.client.deleteFile(remoteFileName);
|
|
||||||
// console.info(`delete ${remoteFileName} succeeded`);
|
|
||||||
} catch (err) {
|
|
||||||
console.error("some error while deleting");
|
|
||||||
console.error(err);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const checkConnectivity = async (
|
|
||||||
client: WrappedWebdavClient,
|
|
||||||
callbackFunc?: any
|
|
||||||
) => {
|
|
||||||
if (
|
|
||||||
!(
|
|
||||||
client.webdavConfig.address.startsWith("http://") ||
|
|
||||||
client.webdavConfig.address.startsWith("https://")
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
const err = "Error: the url should start with http(s):// but it does not!";
|
|
||||||
console.error(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await client.init();
|
|
||||||
const results = await getRemoteMeta(client, `/${client.remoteBaseDir}/`);
|
|
||||||
if (results === undefined) {
|
|
||||||
const err = "results is undefined";
|
|
||||||
console.error(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
} catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
if (callbackFunc !== undefined) {
|
|
||||||
callbackFunc(err);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
150
src/settings.ts
150
src/settings.ts
|
@ -41,17 +41,17 @@ import {
|
||||||
upsertLastSuccessSyncTimeByVault,
|
upsertLastSuccessSyncTimeByVault,
|
||||||
} from "./localdb";
|
} from "./localdb";
|
||||||
import type RemotelySavePlugin from "./main"; // unavoidable
|
import type RemotelySavePlugin from "./main"; // unavoidable
|
||||||
import { RemoteClient } from "./remote";
|
import { FakeFs } from "./fsAll";
|
||||||
import {
|
import {
|
||||||
DEFAULT_DROPBOX_CONFIG,
|
DEFAULT_DROPBOX_CONFIG,
|
||||||
getAuthUrlAndVerifier as getAuthUrlAndVerifierDropbox,
|
getAuthUrlAndVerifier as getAuthUrlAndVerifierDropbox,
|
||||||
sendAuthReq as sendAuthReqDropbox,
|
sendAuthReq as sendAuthReqDropbox,
|
||||||
setConfigBySuccessfullAuthInplace,
|
setConfigBySuccessfullAuthInplace,
|
||||||
} from "./remoteForDropbox";
|
} from "./fsDropbox";
|
||||||
import {
|
import {
|
||||||
DEFAULT_ONEDRIVE_CONFIG,
|
DEFAULT_ONEDRIVE_CONFIG,
|
||||||
getAuthUrlAndVerifier as getAuthUrlAndVerifierOnedrive,
|
getAuthUrlAndVerifier as getAuthUrlAndVerifierOnedrive,
|
||||||
} from "./remoteForOnedrive";
|
} from "./fsOnedrive";
|
||||||
import { messyConfigToNormal } from "./configPersist";
|
import { messyConfigToNormal } from "./configPersist";
|
||||||
import type { TransItemType } from "./i18n";
|
import type { TransItemType } from "./i18n";
|
||||||
import {
|
import {
|
||||||
|
@ -59,8 +59,9 @@ import {
|
||||||
checkHasSpecialCharForDir,
|
checkHasSpecialCharForDir,
|
||||||
stringToFragment,
|
stringToFragment,
|
||||||
} from "./misc";
|
} from "./misc";
|
||||||
import { simpleTransRemotePrefix } from "./remoteForS3";
|
import { simpleTransRemotePrefix } from "./fsS3";
|
||||||
import cloneDeep from "lodash/cloneDeep";
|
import cloneDeep from "lodash/cloneDeep";
|
||||||
|
import { getClient } from "./fsGetter";
|
||||||
|
|
||||||
class PasswordModal extends Modal {
|
class PasswordModal extends Modal {
|
||||||
plugin: RemotelySavePlugin;
|
plugin: RemotelySavePlugin;
|
||||||
|
@ -468,16 +469,12 @@ class DropboxAuthModal extends Modal {
|
||||||
authRes!,
|
authRes!,
|
||||||
() => self.plugin.saveSettings()
|
() => self.plugin.saveSettings()
|
||||||
);
|
);
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"dropbox",
|
this.plugin.settings,
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.plugin.settings.dropbox,
|
|
||||||
undefined,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.plugin.saveSettings()
|
() => this.plugin.saveSettings()
|
||||||
);
|
);
|
||||||
const username = await client.getUser();
|
const username = await client.getUserDisplayName();
|
||||||
this.plugin.settings.dropbox.username = username;
|
this.plugin.settings.dropbox.username = username;
|
||||||
await this.plugin.saveSettings();
|
await this.plugin.saveSettings();
|
||||||
new Notice(
|
new Notice(
|
||||||
|
@ -1077,9 +1074,13 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
||||||
button.setButtonText(t("settings_checkonnectivity_button"));
|
button.setButtonText(t("settings_checkonnectivity_button"));
|
||||||
button.onClick(async () => {
|
button.onClick(async () => {
|
||||||
new Notice(t("settings_checkonnectivity_checking"));
|
new Notice(t("settings_checkonnectivity_checking"));
|
||||||
const client = new RemoteClient("s3", this.plugin.settings.s3);
|
const client = getClient(
|
||||||
|
this.plugin.settings,
|
||||||
|
this.app.vault.getName(),
|
||||||
|
() => this.plugin.saveSettings()
|
||||||
|
);
|
||||||
const errors = { msg: "" };
|
const errors = { msg: "" };
|
||||||
const res = await client.checkConnectivity((err: any) => {
|
const res = await client.checkConnect((err: any) => {
|
||||||
errors.msg = err;
|
errors.msg = err;
|
||||||
});
|
});
|
||||||
if (res) {
|
if (res) {
|
||||||
|
@ -1143,14 +1144,10 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
||||||
button.onClick(async () => {
|
button.onClick(async () => {
|
||||||
try {
|
try {
|
||||||
const self = this;
|
const self = this;
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"dropbox",
|
this.plugin.settings,
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.plugin.settings.dropbox,
|
|
||||||
undefined,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.plugin.saveSettings()
|
() => this.plugin.saveSettings()
|
||||||
);
|
);
|
||||||
await client.revokeAuth();
|
await client.revokeAuth();
|
||||||
this.plugin.settings.dropbox = JSON.parse(
|
this.plugin.settings.dropbox = JSON.parse(
|
||||||
|
@ -1258,18 +1255,14 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
||||||
button.onClick(async () => {
|
button.onClick(async () => {
|
||||||
new Notice(t("settings_checkonnectivity_checking"));
|
new Notice(t("settings_checkonnectivity_checking"));
|
||||||
const self = this;
|
const self = this;
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"dropbox",
|
this.plugin.settings,
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.plugin.settings.dropbox,
|
|
||||||
undefined,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.plugin.saveSettings()
|
() => this.plugin.saveSettings()
|
||||||
);
|
);
|
||||||
|
|
||||||
const errors = { msg: "" };
|
const errors = { msg: "" };
|
||||||
const res = await client.checkConnectivity((err: any) => {
|
const res = await client.checkConnect((err: any) => {
|
||||||
errors.msg = `${err}`;
|
errors.msg = `${err}`;
|
||||||
});
|
});
|
||||||
if (res) {
|
if (res) {
|
||||||
|
@ -1407,18 +1400,13 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
||||||
button.onClick(async () => {
|
button.onClick(async () => {
|
||||||
new Notice(t("settings_checkonnectivity_checking"));
|
new Notice(t("settings_checkonnectivity_checking"));
|
||||||
const self = this;
|
const self = this;
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"onedrive",
|
this.plugin.settings,
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.plugin.settings.onedrive,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.plugin.saveSettings()
|
() => this.plugin.saveSettings()
|
||||||
);
|
);
|
||||||
|
|
||||||
const errors = { msg: "" };
|
const errors = { msg: "" };
|
||||||
const res = await client.checkConnectivity((err: any) => {
|
const res = await client.checkConnect((err: any) => {
|
||||||
errors.msg = `${err}`;
|
errors.msg = `${err}`;
|
||||||
});
|
});
|
||||||
if (res) {
|
if (res) {
|
||||||
|
@ -1617,17 +1605,13 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
||||||
button.onClick(async () => {
|
button.onClick(async () => {
|
||||||
new Notice(t("settings_checkonnectivity_checking"));
|
new Notice(t("settings_checkonnectivity_checking"));
|
||||||
const self = this;
|
const self = this;
|
||||||
const client = new RemoteClient(
|
const client = getClient(
|
||||||
"webdav",
|
this.plugin.settings,
|
||||||
undefined,
|
|
||||||
this.plugin.settings.webdav,
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
this.app.vault.getName(),
|
this.app.vault.getName(),
|
||||||
() => self.plugin.saveSettings()
|
() => this.plugin.saveSettings()
|
||||||
);
|
);
|
||||||
const errors = { msg: "" };
|
const errors = { msg: "" };
|
||||||
const res = await client.checkConnectivity((err: any) => {
|
const res = await client.checkConnect((err: any) => {
|
||||||
errors.msg = `${err}`;
|
errors.msg = `${err}`;
|
||||||
});
|
});
|
||||||
if (res) {
|
if (res) {
|
||||||
|
@ -1790,72 +1774,22 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
|
||||||
});
|
});
|
||||||
|
|
||||||
new Setting(basicDiv)
|
new Setting(basicDiv)
|
||||||
.setName(t("settings_saverun"))
|
.setName(t("settings_synconsave"))
|
||||||
.setDesc(t("settings_saverun_desc"))
|
.setDesc(t("settings_synconsave_desc"))
|
||||||
.addDropdown((dropdown) => {
|
.addDropdown((dropdown) => {
|
||||||
dropdown.addOption("-1", t("settings_saverun_notset"));
|
dropdown.addOption("-1", t("settings_synconsave_disable"));
|
||||||
dropdown.addOption(`${1000 * 1}`, t("settings_saverun_1sec"));
|
dropdown.addOption("1000", t("settings_synconsave_enable"));
|
||||||
dropdown.addOption(`${1000 * 5}`, t("settings_saverun_5sec"));
|
// for backward compatibility, we need to use a number representing seconds
|
||||||
dropdown.addOption(`${1000 * 10}`, t("settings_saverun_10sec"));
|
let syncOnSaveEnabled = false;
|
||||||
dropdown.addOption(`${1000 * 60}`, t("settings_saverun_1min"));
|
if ((this.plugin.settings.syncOnSaveAfterMilliseconds ?? -1) > 0) {
|
||||||
let runScheduled = false;
|
syncOnSaveEnabled = true;
|
||||||
|
}
|
||||||
dropdown
|
dropdown
|
||||||
.setValue(`${this.plugin.settings.syncOnSaveAfterMilliseconds}`)
|
.setValue(`${syncOnSaveEnabled ? "1000" : "-1"}`)
|
||||||
.onChange(async (val: string) => {
|
.onChange(async (val: string) => {
|
||||||
const realVal = parseInt(val);
|
this.plugin.settings.syncOnSaveAfterMilliseconds = parseInt(val);
|
||||||
this.plugin.settings.syncOnSaveAfterMilliseconds = realVal;
|
|
||||||
await this.plugin.saveSettings();
|
await this.plugin.saveSettings();
|
||||||
if (
|
this.plugin.toggleSyncOnSaveIfSet();
|
||||||
(realVal === undefined || realVal === null || realVal <= 0) &&
|
|
||||||
this.plugin.syncOnSaveIntervalID !== undefined
|
|
||||||
) {
|
|
||||||
// clear
|
|
||||||
window.clearInterval(this.plugin.syncOnSaveIntervalID);
|
|
||||||
this.plugin.syncOnSaveIntervalID = undefined;
|
|
||||||
} else if (
|
|
||||||
realVal !== undefined &&
|
|
||||||
realVal !== null &&
|
|
||||||
realVal > 0
|
|
||||||
) {
|
|
||||||
const intervalID = window.setInterval(() => {
|
|
||||||
const currentFile = this.app.workspace.getActiveFile();
|
|
||||||
|
|
||||||
if (currentFile) {
|
|
||||||
// get the last modified time of the current file
|
|
||||||
// if it has been modified within the last syncOnSaveAfterMilliseconds
|
|
||||||
// then schedule a run for syncOnSaveAfterMilliseconds after it was modified
|
|
||||||
const lastModified = currentFile.stat.mtime;
|
|
||||||
const currentTime = Date.now();
|
|
||||||
// console.debug(
|
|
||||||
// `Checking if file was modified within last ${
|
|
||||||
// this.plugin.settings.syncOnSaveAfterMilliseconds / 1000
|
|
||||||
// } seconds, last modified: ${
|
|
||||||
// (currentTime - lastModified) / 1000
|
|
||||||
// } seconds ago`
|
|
||||||
// );
|
|
||||||
if (
|
|
||||||
currentTime - lastModified <
|
|
||||||
this.plugin.settings.syncOnSaveAfterMilliseconds!
|
|
||||||
) {
|
|
||||||
if (!runScheduled) {
|
|
||||||
const scheduleTimeFromNow =
|
|
||||||
this.plugin.settings.syncOnSaveAfterMilliseconds! -
|
|
||||||
(currentTime - lastModified);
|
|
||||||
console.info(
|
|
||||||
`schedule a run for ${scheduleTimeFromNow} milliseconds later`
|
|
||||||
);
|
|
||||||
runScheduled = true;
|
|
||||||
setTimeout(() => {
|
|
||||||
this.plugin.syncRun("auto_sync_on_save");
|
|
||||||
runScheduled = false;
|
|
||||||
}, scheduleTimeFromNow);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, realVal);
|
|
||||||
this.plugin.syncOnSaveIntervalID = intervalID;
|
|
||||||
this.plugin.registerInterval(intervalID);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
728
src/sync.ts
728
src/sync.ts
|
@ -1,157 +1,38 @@
|
||||||
import PQueue from "p-queue";
|
|
||||||
import XRegExp from "xregexp";
|
import XRegExp from "xregexp";
|
||||||
import type {
|
import {
|
||||||
CipherMethodType,
|
|
||||||
ConflictActionType,
|
ConflictActionType,
|
||||||
EmptyFolderCleanType,
|
EmptyFolderCleanType,
|
||||||
Entity,
|
Entity,
|
||||||
MixedEntity,
|
MixedEntity,
|
||||||
|
RemotelySavePluginSettings,
|
||||||
SUPPORTED_SERVICES_TYPE,
|
SUPPORTED_SERVICES_TYPE,
|
||||||
SyncDirectionType,
|
SyncDirectionType,
|
||||||
|
SyncTriggerSourceType,
|
||||||
} from "./baseTypes";
|
} from "./baseTypes";
|
||||||
import { isInsideObsFolder } from "./obsFolderLister";
|
import { FakeFs } from "./fsAll";
|
||||||
|
import { FakeFsEncrypt } from "./fsEncrypt";
|
||||||
|
import {
|
||||||
|
InternalDBs,
|
||||||
|
clearPrevSyncRecordByVaultAndProfile,
|
||||||
|
getAllPrevSyncRecordsByVaultAndProfile,
|
||||||
|
insertSyncPlanRecordByVault,
|
||||||
|
upsertPrevSyncRecordByVaultAndProfile,
|
||||||
|
} from "./localdb";
|
||||||
import {
|
import {
|
||||||
isSpecialFolderNameToSkip,
|
|
||||||
isHiddenPath,
|
|
||||||
unixTimeToStr,
|
|
||||||
getParentFolder,
|
|
||||||
isVaildText,
|
|
||||||
atWhichLevel,
|
atWhichLevel,
|
||||||
mkdirpInVault,
|
|
||||||
getFolderLevels,
|
getFolderLevels,
|
||||||
|
getParentFolder,
|
||||||
|
isHiddenPath,
|
||||||
|
isSpecialFolderNameToSkip,
|
||||||
|
unixTimeToStr,
|
||||||
} from "./misc";
|
} from "./misc";
|
||||||
|
import { Profiler } from "./profiler";
|
||||||
import {
|
import {
|
||||||
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
|
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
|
||||||
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2,
|
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2,
|
||||||
} from "./metadataOnRemote";
|
} from "./metadataOnRemote";
|
||||||
import { RemoteClient } from "./remote";
|
|
||||||
import { Vault } from "obsidian";
|
|
||||||
|
|
||||||
import AggregateError from "aggregate-error";
|
import AggregateError from "aggregate-error";
|
||||||
import {
|
import PQueue from "p-queue";
|
||||||
InternalDBs,
|
|
||||||
clearPrevSyncRecordByVaultAndProfile,
|
|
||||||
upsertPrevSyncRecordByVaultAndProfile,
|
|
||||||
} from "./localdb";
|
|
||||||
import { Cipher } from "./encryptUnified";
|
|
||||||
import { Profiler } from "./profiler";
|
|
||||||
|
|
||||||
export type SyncStatusType =
|
|
||||||
| "idle"
|
|
||||||
| "preparing"
|
|
||||||
| "getting_remote_files_list"
|
|
||||||
| "getting_local_meta"
|
|
||||||
| "getting_local_prev_sync"
|
|
||||||
| "checking_password"
|
|
||||||
| "generating_plan"
|
|
||||||
| "syncing"
|
|
||||||
| "cleaning"
|
|
||||||
| "finish";
|
|
||||||
|
|
||||||
export interface PasswordCheckType {
|
|
||||||
ok: boolean;
|
|
||||||
reason:
|
|
||||||
| "empty_remote"
|
|
||||||
| "unknown_encryption_method"
|
|
||||||
| "remote_encrypted_local_no_password"
|
|
||||||
| "password_matched"
|
|
||||||
| "password_or_method_not_matched_or_remote_not_encrypted"
|
|
||||||
| "likely_no_password_both_sides"
|
|
||||||
| "encryption_method_not_matched";
|
|
||||||
}
|
|
||||||
|
|
||||||
export const isPasswordOk = async (
|
|
||||||
remote: Entity[],
|
|
||||||
cipher: Cipher
|
|
||||||
): Promise<PasswordCheckType> => {
|
|
||||||
if (remote === undefined || remote.length === 0) {
|
|
||||||
// remote empty
|
|
||||||
return {
|
|
||||||
ok: true,
|
|
||||||
reason: "empty_remote",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
const santyCheckKey = remote[0].keyRaw;
|
|
||||||
|
|
||||||
if (cipher.isPasswordEmpty()) {
|
|
||||||
// TODO: no way to distinguish remote rclone encrypted
|
|
||||||
// if local has no password??
|
|
||||||
if (Cipher.isLikelyEncryptedName(santyCheckKey)) {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
reason: "remote_encrypted_local_no_password",
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
return {
|
|
||||||
ok: true,
|
|
||||||
reason: "likely_no_password_both_sides",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (cipher.method === "unknown") {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
reason: "unknown_encryption_method",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
Cipher.isLikelyEncryptedNameNotMatchMethod(santyCheckKey, cipher.method)
|
|
||||||
) {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
reason: "encryption_method_not_matched",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const k = await cipher.decryptName(santyCheckKey);
|
|
||||||
if (k === undefined) {
|
|
||||||
throw Error(`decryption failed`);
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
ok: true,
|
|
||||||
reason: "password_matched",
|
|
||||||
};
|
|
||||||
} catch (error) {
|
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
reason: "password_or_method_not_matched_or_remote_not_encrypted",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const isSkipItemByName = (
|
|
||||||
key: string,
|
|
||||||
syncConfigDir: boolean,
|
|
||||||
syncUnderscoreItems: boolean,
|
|
||||||
configDir: string,
|
|
||||||
ignorePaths: string[]
|
|
||||||
) => {
|
|
||||||
if (key === undefined) {
|
|
||||||
throw Error(`isSkipItemByName meets undefinded key!`);
|
|
||||||
}
|
|
||||||
if (ignorePaths !== undefined && ignorePaths.length > 0) {
|
|
||||||
for (const r of ignorePaths) {
|
|
||||||
if (XRegExp(r, "A").test(key)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (syncConfigDir && isInsideObsFolder(key, configDir)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (isSpecialFolderNameToSkip(key, [])) {
|
|
||||||
// some special dirs and files are always skipped
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return (
|
|
||||||
isHiddenPath(key, true, false) ||
|
|
||||||
(!syncUnderscoreItems && isHiddenPath(key, false, true)) ||
|
|
||||||
key === "/" ||
|
|
||||||
key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE ||
|
|
||||||
key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
const copyEntityAndFixTimeFormat = (
|
const copyEntityAndFixTimeFormat = (
|
||||||
src: Entity,
|
src: Entity,
|
||||||
|
@ -195,52 +76,6 @@ const copyEntityAndFixTimeFormat = (
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
|
||||||
* Inplace, no copy again.
|
|
||||||
*/
|
|
||||||
const decryptRemoteEntityInplace = async (remote: Entity, cipher: Cipher) => {
|
|
||||||
if (cipher?.isPasswordEmpty()) {
|
|
||||||
remote.key = remote.keyRaw;
|
|
||||||
remote.keyEnc = remote.keyRaw;
|
|
||||||
remote.size = remote.sizeRaw;
|
|
||||||
remote.sizeEnc = remote.sizeRaw;
|
|
||||||
return remote;
|
|
||||||
}
|
|
||||||
|
|
||||||
remote.keyEnc = remote.keyRaw;
|
|
||||||
remote.key = await cipher.decryptName(remote.keyEnc);
|
|
||||||
remote.sizeEnc = remote.sizeRaw;
|
|
||||||
|
|
||||||
// TODO
|
|
||||||
// remote.size = getSizeFromEncToOrig(remote.sizeEnc, password);
|
|
||||||
// but we don't have deterministic way to get a number because the encryption has padding...
|
|
||||||
|
|
||||||
return remote;
|
|
||||||
};
|
|
||||||
|
|
||||||
const fullfillMTimeOfRemoteEntityInplace = (
|
|
||||||
remote: Entity,
|
|
||||||
mtimeCli?: number
|
|
||||||
) => {
|
|
||||||
// TODO:
|
|
||||||
// on 20240405, we find that dropbox's mtimeCli is not updated
|
|
||||||
// if the content is not updated even the time is updated...
|
|
||||||
// so we do not check remote.mtimeCli for now..
|
|
||||||
if (
|
|
||||||
mtimeCli !== undefined &&
|
|
||||||
mtimeCli > 0 /* &&
|
|
||||||
(remote.mtimeCli === undefined ||
|
|
||||||
remote.mtimeCli <= 0 ||
|
|
||||||
(remote.mtimeSvr !== undefined &&
|
|
||||||
remote.mtimeSvr > 0 &&
|
|
||||||
remote.mtimeCli >= remote.mtimeSvr))
|
|
||||||
*/
|
|
||||||
) {
|
|
||||||
remote.mtimeCli = mtimeCli;
|
|
||||||
}
|
|
||||||
return remote;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Directly throw error here.
|
* Directly throw error here.
|
||||||
* We can only defer the checking now, because before decryption we don't know whether it's a file or folder.
|
* We can only defer the checking now, because before decryption we don't know whether it's a file or folder.
|
||||||
|
@ -265,62 +100,49 @@ const ensureMTimeOfRemoteEntityValid = (remote: Entity) => {
|
||||||
return remote;
|
return remote;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
const isInsideObsFolder = (x: string, configDir: string) => {
|
||||||
* Inplace, no copy again.
|
if (!configDir.startsWith(".")) {
|
||||||
*/
|
throw Error(`configDir should starts with . but we get ${configDir}`);
|
||||||
const encryptLocalEntityInplace = async (
|
}
|
||||||
local: Entity,
|
return x === configDir || x.startsWith(`${configDir}/`);
|
||||||
cipher: Cipher,
|
};
|
||||||
remoteKeyEnc: string | undefined
|
|
||||||
|
const isSkipItemByName = (
|
||||||
|
key: string,
|
||||||
|
syncConfigDir: boolean,
|
||||||
|
syncUnderscoreItems: boolean,
|
||||||
|
configDir: string,
|
||||||
|
ignorePaths: string[]
|
||||||
) => {
|
) => {
|
||||||
// console.debug(
|
if (key === undefined) {
|
||||||
// `encryptLocalEntityInplace: local=${JSON.stringify(
|
throw Error(`isSkipItemByName meets undefinded key!`);
|
||||||
// local,
|
|
||||||
// null,
|
|
||||||
// 2
|
|
||||||
// )}, password=${
|
|
||||||
// password === undefined || password === "" ? "[empty]" : "[not empty]"
|
|
||||||
// }, remoteKeyEnc=${remoteKeyEnc}`
|
|
||||||
// );
|
|
||||||
|
|
||||||
if (local.key === undefined) {
|
|
||||||
// local.key should always have value
|
|
||||||
throw Error(`local ${local.keyRaw} is abnormal without key`);
|
|
||||||
}
|
}
|
||||||
|
if (ignorePaths !== undefined && ignorePaths.length > 0) {
|
||||||
if (cipher.isPasswordEmpty()) {
|
for (const r of ignorePaths) {
|
||||||
local.sizeEnc = local.sizeRaw; // if no enc, the remote file has the same size
|
if (XRegExp(r, "A").test(key)) {
|
||||||
local.keyEnc = local.keyRaw;
|
return true;
|
||||||
return local;
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// below is for having password
|
|
||||||
if (local.sizeEnc === undefined && local.size !== undefined) {
|
|
||||||
// it's not filled yet, we fill it
|
|
||||||
// local.size is possibly undefined if it's "prevSync" Entity
|
|
||||||
// but local.key should always have value
|
|
||||||
local.sizeEnc = cipher.getSizeFromOrigToEnc(local.size);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (local.keyEnc === undefined || local.keyEnc === "") {
|
|
||||||
if (
|
|
||||||
remoteKeyEnc !== undefined &&
|
|
||||||
remoteKeyEnc !== "" &&
|
|
||||||
remoteKeyEnc !== local.key
|
|
||||||
) {
|
|
||||||
// we can reuse remote encrypted key if any
|
|
||||||
local.keyEnc = remoteKeyEnc;
|
|
||||||
} else {
|
|
||||||
// we assign a new encrypted key because of no remote
|
|
||||||
local.keyEnc = await cipher.encryptName(local.key);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return local;
|
if (syncConfigDir && isInsideObsFolder(key, configDir)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (isSpecialFolderNameToSkip(key, [])) {
|
||||||
|
// some special dirs and files are always skipped
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
isHiddenPath(key, true, false) ||
|
||||||
|
(!syncUnderscoreItems && isHiddenPath(key, false, true)) ||
|
||||||
|
key === "/" ||
|
||||||
|
key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE ||
|
||||||
|
key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2
|
||||||
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
export type SyncPlanType = Record<string, MixedEntity>;
|
export type SyncPlanType = Record<string, MixedEntity>;
|
||||||
|
|
||||||
export const ensembleMixedEnties = async (
|
const ensembleMixedEnties = async (
|
||||||
localEntityList: Entity[],
|
localEntityList: Entity[],
|
||||||
prevSyncEntityList: Entity[],
|
prevSyncEntityList: Entity[],
|
||||||
remoteEntityList: Entity[],
|
remoteEntityList: Entity[],
|
||||||
|
@ -329,7 +151,7 @@ export const ensembleMixedEnties = async (
|
||||||
configDir: string,
|
configDir: string,
|
||||||
syncUnderscoreItems: boolean,
|
syncUnderscoreItems: boolean,
|
||||||
ignorePaths: string[],
|
ignorePaths: string[],
|
||||||
cipher: Cipher,
|
fsEncrypt: FakeFsEncrypt,
|
||||||
serviceType: SUPPORTED_SERVICES_TYPE,
|
serviceType: SUPPORTED_SERVICES_TYPE,
|
||||||
|
|
||||||
profiler: Profiler
|
profiler: Profiler
|
||||||
|
@ -345,10 +167,7 @@ export const ensembleMixedEnties = async (
|
||||||
// we also have to synthesize folders here
|
// we also have to synthesize folders here
|
||||||
for (const remote of remoteEntityList) {
|
for (const remote of remoteEntityList) {
|
||||||
const remoteCopied = ensureMTimeOfRemoteEntityValid(
|
const remoteCopied = ensureMTimeOfRemoteEntityValid(
|
||||||
await decryptRemoteEntityInplace(
|
copyEntityAndFixTimeFormat(remote, serviceType)
|
||||||
copyEntityAndFixTimeFormat(remote, serviceType),
|
|
||||||
cipher
|
|
||||||
)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const key = remoteCopied.key!;
|
const key = remoteCopied.key!;
|
||||||
|
@ -433,19 +252,13 @@ export const ensembleMixedEnties = async (
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: abstraction leaking?
|
||||||
|
const prevSyncCopied = await fsEncrypt.encryptEntity(
|
||||||
|
copyEntityAndFixTimeFormat(prevSync, serviceType)
|
||||||
|
);
|
||||||
if (finalMappings.hasOwnProperty(key)) {
|
if (finalMappings.hasOwnProperty(key)) {
|
||||||
const prevSyncCopied = await encryptLocalEntityInplace(
|
|
||||||
copyEntityAndFixTimeFormat(prevSync, serviceType),
|
|
||||||
cipher,
|
|
||||||
finalMappings[key].remote?.keyEnc
|
|
||||||
);
|
|
||||||
finalMappings[key].prevSync = prevSyncCopied;
|
finalMappings[key].prevSync = prevSyncCopied;
|
||||||
} else {
|
} else {
|
||||||
const prevSyncCopied = await encryptLocalEntityInplace(
|
|
||||||
copyEntityAndFixTimeFormat(prevSync, serviceType),
|
|
||||||
cipher,
|
|
||||||
undefined
|
|
||||||
);
|
|
||||||
finalMappings[key] = {
|
finalMappings[key] = {
|
||||||
key: key,
|
key: key,
|
||||||
prevSync: prevSyncCopied,
|
prevSync: prevSyncCopied,
|
||||||
|
@ -473,19 +286,13 @@ export const ensembleMixedEnties = async (
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: abstraction leaking?
|
||||||
|
const localCopied = await fsEncrypt.encryptEntity(
|
||||||
|
copyEntityAndFixTimeFormat(local, serviceType)
|
||||||
|
);
|
||||||
if (finalMappings.hasOwnProperty(key)) {
|
if (finalMappings.hasOwnProperty(key)) {
|
||||||
const localCopied = await encryptLocalEntityInplace(
|
|
||||||
copyEntityAndFixTimeFormat(local, serviceType),
|
|
||||||
cipher,
|
|
||||||
finalMappings[key].remote?.keyEnc
|
|
||||||
);
|
|
||||||
finalMappings[key].local = localCopied;
|
finalMappings[key].local = localCopied;
|
||||||
} else {
|
} else {
|
||||||
const localCopied = await encryptLocalEntityInplace(
|
|
||||||
copyEntityAndFixTimeFormat(local, serviceType),
|
|
||||||
cipher,
|
|
||||||
undefined
|
|
||||||
);
|
|
||||||
finalMappings[key] = {
|
finalMappings[key] = {
|
||||||
key: key,
|
key: key,
|
||||||
local: localCopied,
|
local: localCopied,
|
||||||
|
@ -508,7 +315,7 @@ export const ensembleMixedEnties = async (
|
||||||
* Basically follow the sync algorithm of https://github.com/Jwink3101/syncrclone
|
* Basically follow the sync algorithm of https://github.com/Jwink3101/syncrclone
|
||||||
* Also deal with syncDirection which makes it more complicated
|
* Also deal with syncDirection which makes it more complicated
|
||||||
*/
|
*/
|
||||||
export const getSyncPlanInplace = async (
|
const getSyncPlanInplace = async (
|
||||||
mixedEntityMappings: Record<string, MixedEntity>,
|
mixedEntityMappings: Record<string, MixedEntity>,
|
||||||
howToCleanEmptyFolder: EmptyFolderCleanType,
|
howToCleanEmptyFolder: EmptyFolderCleanType,
|
||||||
skipSizeLargerThan: number,
|
skipSizeLargerThan: number,
|
||||||
|
@ -940,6 +747,7 @@ export const getSyncPlanInplace = async (
|
||||||
mixedEntityMappings["/$@meta"] = {
|
mixedEntityMappings["/$@meta"] = {
|
||||||
key: "/$@meta", // don't mess up with the types
|
key: "/$@meta", // don't mess up with the types
|
||||||
sideNotes: {
|
sideNotes: {
|
||||||
|
version: "2024047 fs version",
|
||||||
generateTime: currTime,
|
generateTime: currTime,
|
||||||
generateTimeFmt: currTimeFmt,
|
generateTimeFmt: currTimeFmt,
|
||||||
},
|
},
|
||||||
|
@ -1093,16 +901,96 @@ const splitFourStepsOnEntityMappings = (
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const fullfillMTimeOfRemoteEntityInplace = (
|
||||||
|
remote: Entity,
|
||||||
|
mtimeCli?: number
|
||||||
|
) => {
|
||||||
|
// TODO:
|
||||||
|
// on 20240405, we find that dropbox's mtimeCli is not updated
|
||||||
|
// if the content is not updated even the time is updated...
|
||||||
|
// so we do not check remote.mtimeCli for now..
|
||||||
|
if (
|
||||||
|
mtimeCli !== undefined &&
|
||||||
|
mtimeCli > 0 /* &&
|
||||||
|
(remote.mtimeCli === undefined ||
|
||||||
|
remote.mtimeCli <= 0 ||
|
||||||
|
(remote.mtimeSvr !== undefined &&
|
||||||
|
remote.mtimeSvr > 0 &&
|
||||||
|
remote.mtimeCli >= remote.mtimeSvr))
|
||||||
|
*/
|
||||||
|
) {
|
||||||
|
remote.mtimeCli = mtimeCli;
|
||||||
|
}
|
||||||
|
return remote;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function copyFolder(
|
||||||
|
key: string,
|
||||||
|
left: FakeFs,
|
||||||
|
right: FakeFs
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (!key.endsWith("/")) {
|
||||||
|
throw Error(`should not call ${key} in copyFolder`);
|
||||||
|
}
|
||||||
|
const statsLeft = await left.stat(key);
|
||||||
|
return await right.mkdir(key, statsLeft.mtimeCli);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function copyFile(
|
||||||
|
key: string,
|
||||||
|
left: FakeFs,
|
||||||
|
right: FakeFs
|
||||||
|
): Promise<Entity> {
|
||||||
|
// console.debug(`copyFile: key=${key}, left=${left.kind}, right=${right.kind}`);
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
throw Error(`should not call ${key} in copyFile`);
|
||||||
|
}
|
||||||
|
const statsLeft = await left.stat(key);
|
||||||
|
const content = await left.readFile(key);
|
||||||
|
|
||||||
|
if (statsLeft.size === undefined) {
|
||||||
|
statsLeft.size = content.byteLength;
|
||||||
|
} else {
|
||||||
|
if (statsLeft.size !== content.byteLength) {
|
||||||
|
throw Error(
|
||||||
|
`error copying ${left.kind}=>${right.kind}: size not matched`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (statsLeft.mtimeCli === undefined) {
|
||||||
|
throw Error(`error copying ${left.kind}=>${right.kind}, no mtimeCli`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// console.debug(`copyFile: about to start right.writeFile`);
|
||||||
|
return await right.writeFile(
|
||||||
|
key,
|
||||||
|
content,
|
||||||
|
statsLeft.mtimeCli,
|
||||||
|
statsLeft.mtimeCli /* TODO */
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function copyFileOrFolder(
|
||||||
|
key: string,
|
||||||
|
left: FakeFs,
|
||||||
|
right: FakeFs
|
||||||
|
): Promise<Entity> {
|
||||||
|
if (key.endsWith("/")) {
|
||||||
|
return await copyFolder(key, left, right);
|
||||||
|
} else {
|
||||||
|
return await copyFile(key, left, right);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const dispatchOperationToActualV3 = async (
|
const dispatchOperationToActualV3 = async (
|
||||||
key: string,
|
key: string,
|
||||||
vaultRandomID: string,
|
vaultRandomID: string,
|
||||||
profileID: string,
|
profileID: string,
|
||||||
r: MixedEntity,
|
r: MixedEntity,
|
||||||
client: RemoteClient,
|
fsLocal: FakeFs,
|
||||||
db: InternalDBs,
|
fsEncrypt: FakeFsEncrypt,
|
||||||
vault: Vault,
|
db: InternalDBs
|
||||||
localDeleteFunc: any,
|
|
||||||
cipher: Cipher
|
|
||||||
) => {
|
) => {
|
||||||
// console.debug(
|
// console.debug(
|
||||||
// `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify(
|
// `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify(
|
||||||
|
@ -1136,11 +1024,8 @@ const dispatchOperationToActualV3 = async (
|
||||||
// if we don't have prevSync, we use remote entity AND local mtime
|
// if we don't have prevSync, we use remote entity AND local mtime
|
||||||
// as if it is "uploaded"
|
// as if it is "uploaded"
|
||||||
if (r.remote !== undefined) {
|
if (r.remote !== undefined) {
|
||||||
let entity = await decryptRemoteEntityInplace(r.remote, cipher);
|
let entity = r.remote;
|
||||||
entity = await fullfillMTimeOfRemoteEntityInplace(
|
entity = fullfillMTimeOfRemoteEntityInplace(entity, r.local?.mtimeCli);
|
||||||
entity,
|
|
||||||
r.local?.mtimeCli
|
|
||||||
);
|
|
||||||
|
|
||||||
if (entity !== undefined) {
|
if (entity !== undefined) {
|
||||||
await upsertPrevSyncRecordByVaultAndProfile(
|
await upsertPrevSyncRecordByVaultAndProfile(
|
||||||
|
@ -1159,38 +1044,17 @@ const dispatchOperationToActualV3 = async (
|
||||||
r.decision === "conflict_created_then_keep_local" ||
|
r.decision === "conflict_created_then_keep_local" ||
|
||||||
r.decision === "conflict_modified_then_keep_local"
|
r.decision === "conflict_modified_then_keep_local"
|
||||||
) {
|
) {
|
||||||
if (
|
// console.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`);
|
||||||
client.serviceType === "onedrive" &&
|
const mtimeCli = (await fsLocal.stat(r.key)).mtimeCli!;
|
||||||
r.local!.size === 0 &&
|
const entity = await copyFileOrFolder(r.key, fsLocal, fsEncrypt);
|
||||||
cipher.isPasswordEmpty()
|
fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli);
|
||||||
) {
|
// console.debug(`after fullfill, entity=${JSON.stringify(entity,null,2)}`)
|
||||||
// special treatment for empty files for OneDrive
|
await upsertPrevSyncRecordByVaultAndProfile(
|
||||||
// TODO: it's ugly, any other way?
|
db,
|
||||||
// special treatment for OneDrive: do nothing, skip empty file without encryption
|
vaultRandomID,
|
||||||
// if it's empty folder, or it's encrypted file/folder, it continues to be uploaded.
|
profileID,
|
||||||
} else {
|
entity
|
||||||
// console.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`);
|
);
|
||||||
const { entity, mtimeCli } = await client.uploadToRemote(
|
|
||||||
r.key,
|
|
||||||
vault,
|
|
||||||
false,
|
|
||||||
cipher,
|
|
||||||
r.local!.keyEnc
|
|
||||||
);
|
|
||||||
// console.debug(`after uploadToRemote`);
|
|
||||||
// console.debug(`entity=${JSON.stringify(entity,null,2)}`)
|
|
||||||
// console.debug(`mtimeCli=${mtimeCli}`)
|
|
||||||
await decryptRemoteEntityInplace(entity, cipher);
|
|
||||||
// console.debug(`after dec, entity=${JSON.stringify(entity,null,2)}`)
|
|
||||||
await fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli);
|
|
||||||
// console.debug(`after fullfill, entity=${JSON.stringify(entity,null,2)}`)
|
|
||||||
await upsertPrevSyncRecordByVaultAndProfile(
|
|
||||||
db,
|
|
||||||
vaultRandomID,
|
|
||||||
profileID,
|
|
||||||
entity
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else if (
|
} else if (
|
||||||
r.decision === "remote_is_modified_then_pull" ||
|
r.decision === "remote_is_modified_then_pull" ||
|
||||||
r.decision === "remote_is_created_then_pull" ||
|
r.decision === "remote_is_created_then_pull" ||
|
||||||
|
@ -1198,14 +1062,11 @@ const dispatchOperationToActualV3 = async (
|
||||||
r.decision === "conflict_modified_then_keep_remote" ||
|
r.decision === "conflict_modified_then_keep_remote" ||
|
||||||
r.decision === "folder_existed_remote_then_also_create_local"
|
r.decision === "folder_existed_remote_then_also_create_local"
|
||||||
) {
|
) {
|
||||||
await mkdirpInVault(r.key, vault);
|
if (r.key.endsWith("/")) {
|
||||||
await client.downloadFromRemote(
|
await fsLocal.mkdir(r.key);
|
||||||
r.key,
|
} else {
|
||||||
vault,
|
await copyFile(r.key, fsEncrypt, fsLocal);
|
||||||
r.remote!.mtimeCli!,
|
}
|
||||||
cipher,
|
|
||||||
r.remote!.keyEnc
|
|
||||||
);
|
|
||||||
await upsertPrevSyncRecordByVaultAndProfile(
|
await upsertPrevSyncRecordByVaultAndProfile(
|
||||||
db,
|
db,
|
||||||
vaultRandomID,
|
vaultRandomID,
|
||||||
|
@ -1214,12 +1075,7 @@ const dispatchOperationToActualV3 = async (
|
||||||
);
|
);
|
||||||
} else if (r.decision === "local_is_deleted_thus_also_delete_remote") {
|
} else if (r.decision === "local_is_deleted_thus_also_delete_remote") {
|
||||||
// local is deleted, we need to delete remote now
|
// local is deleted, we need to delete remote now
|
||||||
await client.deleteFromRemote(
|
await fsEncrypt.rm(r.key);
|
||||||
r.key,
|
|
||||||
cipher,
|
|
||||||
r.remote!.keyEnc,
|
|
||||||
r.remote!.synthesizedFolder
|
|
||||||
);
|
|
||||||
await clearPrevSyncRecordByVaultAndProfile(
|
await clearPrevSyncRecordByVaultAndProfile(
|
||||||
db,
|
db,
|
||||||
vaultRandomID,
|
vaultRandomID,
|
||||||
|
@ -1228,7 +1084,7 @@ const dispatchOperationToActualV3 = async (
|
||||||
);
|
);
|
||||||
} else if (r.decision === "remote_is_deleted_thus_also_delete_local") {
|
} else if (r.decision === "remote_is_deleted_thus_also_delete_local") {
|
||||||
// remote is deleted, we need to delete local now
|
// remote is deleted, we need to delete local now
|
||||||
await localDeleteFunc(r.key);
|
await fsLocal.rm(r.key);
|
||||||
await clearPrevSyncRecordByVaultAndProfile(
|
await clearPrevSyncRecordByVaultAndProfile(
|
||||||
db,
|
db,
|
||||||
vaultRandomID,
|
vaultRandomID,
|
||||||
|
@ -1241,17 +1097,8 @@ const dispatchOperationToActualV3 = async (
|
||||||
) {
|
) {
|
||||||
throw Error(`${r.decision} not implemented yet: ${JSON.stringify(r)}`);
|
throw Error(`${r.decision} not implemented yet: ${JSON.stringify(r)}`);
|
||||||
} else if (r.decision === "folder_to_be_created") {
|
} else if (r.decision === "folder_to_be_created") {
|
||||||
await mkdirpInVault(r.key, vault);
|
await fsLocal.mkdir(r.key);
|
||||||
const { entity, mtimeCli } = await client.uploadToRemote(
|
const entity = await copyFolder(r.key, fsLocal, fsEncrypt);
|
||||||
r.key,
|
|
||||||
vault,
|
|
||||||
false,
|
|
||||||
cipher,
|
|
||||||
r.local!.keyEnc
|
|
||||||
);
|
|
||||||
// we need to decrypt the key!!!
|
|
||||||
await decryptRemoteEntityInplace(entity, cipher);
|
|
||||||
await fullfillMTimeOfRemoteEntityInplace(entity, mtimeCli);
|
|
||||||
await upsertPrevSyncRecordByVaultAndProfile(
|
await upsertPrevSyncRecordByVaultAndProfile(
|
||||||
db,
|
db,
|
||||||
vaultRandomID,
|
vaultRandomID,
|
||||||
|
@ -1267,18 +1114,13 @@ const dispatchOperationToActualV3 = async (
|
||||||
r.decision === "folder_to_be_deleted_on_both" ||
|
r.decision === "folder_to_be_deleted_on_both" ||
|
||||||
r.decision === "folder_to_be_deleted_on_local"
|
r.decision === "folder_to_be_deleted_on_local"
|
||||||
) {
|
) {
|
||||||
await localDeleteFunc(r.key);
|
await fsLocal.rm(r.key);
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
r.decision === "folder_to_be_deleted_on_both" ||
|
r.decision === "folder_to_be_deleted_on_both" ||
|
||||||
r.decision === "folder_to_be_deleted_on_remote"
|
r.decision === "folder_to_be_deleted_on_remote"
|
||||||
) {
|
) {
|
||||||
await client.deleteFromRemote(
|
await fsEncrypt.rm(r.key);
|
||||||
r.key,
|
|
||||||
cipher,
|
|
||||||
r.remote!.keyEnc,
|
|
||||||
r.remote!.synthesizedFolder
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
await clearPrevSyncRecordByVaultAndProfile(
|
await clearPrevSyncRecordByVaultAndProfile(
|
||||||
db,
|
db,
|
||||||
|
@ -1293,18 +1135,16 @@ const dispatchOperationToActualV3 = async (
|
||||||
|
|
||||||
export const doActualSync = async (
|
export const doActualSync = async (
|
||||||
mixedEntityMappings: Record<string, MixedEntity>,
|
mixedEntityMappings: Record<string, MixedEntity>,
|
||||||
client: RemoteClient,
|
fsLocal: FakeFs,
|
||||||
|
fsEncrypt: FakeFsEncrypt,
|
||||||
vaultRandomID: string,
|
vaultRandomID: string,
|
||||||
profileID: string,
|
profileID: string,
|
||||||
vault: Vault,
|
|
||||||
cipher: Cipher,
|
|
||||||
concurrency: number,
|
concurrency: number,
|
||||||
localDeleteFunc: any,
|
|
||||||
protectModifyPercentage: number,
|
protectModifyPercentage: number,
|
||||||
getProtectModifyPercentageErrorStrFunc: any,
|
getProtectModifyPercentageErrorStrFunc: any,
|
||||||
callbackSyncProcess: any,
|
|
||||||
db: InternalDBs,
|
db: InternalDBs,
|
||||||
profiler: Profiler
|
profiler: Profiler,
|
||||||
|
callbackSyncProcess?: any
|
||||||
) => {
|
) => {
|
||||||
profiler.addIndent();
|
profiler.addIndent();
|
||||||
profiler.insert("doActualSync: enter");
|
profiler.insert("doActualSync: enter");
|
||||||
|
@ -1400,27 +1240,23 @@ export const doActualSync = async (
|
||||||
// `start syncing "${key}" with plan ${JSON.stringify(val)}`
|
// `start syncing "${key}" with plan ${JSON.stringify(val)}`
|
||||||
// );
|
// );
|
||||||
|
|
||||||
if (callbackSyncProcess !== undefined) {
|
await callbackSyncProcess?.(
|
||||||
await callbackSyncProcess(
|
realCounter,
|
||||||
realCounter,
|
realTotalCount,
|
||||||
realTotalCount,
|
key,
|
||||||
key,
|
val.decision
|
||||||
val.decision
|
);
|
||||||
);
|
|
||||||
|
|
||||||
realCounter += 1;
|
realCounter += 1;
|
||||||
}
|
|
||||||
|
|
||||||
await dispatchOperationToActualV3(
|
await dispatchOperationToActualV3(
|
||||||
key,
|
key,
|
||||||
vaultRandomID,
|
vaultRandomID,
|
||||||
profileID,
|
profileID,
|
||||||
val,
|
val,
|
||||||
client,
|
fsLocal,
|
||||||
db,
|
fsEncrypt,
|
||||||
vault,
|
db
|
||||||
localDeleteFunc,
|
|
||||||
cipher
|
|
||||||
);
|
);
|
||||||
|
|
||||||
// console.debug(`finished ${key}`);
|
// console.debug(`finished ${key}`);
|
||||||
|
@ -1456,3 +1292,191 @@ export const doActualSync = async (
|
||||||
profiler.insert(`doActualSync: exit`);
|
profiler.insert(`doActualSync: exit`);
|
||||||
profiler.removeIndent();
|
profiler.removeIndent();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type SyncStatusType =
|
||||||
|
| "idle"
|
||||||
|
| "preparing"
|
||||||
|
| "getting_remote_files_list"
|
||||||
|
| "getting_local_meta"
|
||||||
|
| "getting_local_prev_sync"
|
||||||
|
| "checking_password"
|
||||||
|
| "generating_plan"
|
||||||
|
| "syncing"
|
||||||
|
| "cleaning"
|
||||||
|
| "finish";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every input variable should be mockable, so that testable.
|
||||||
|
*/
|
||||||
|
export async function syncer(
|
||||||
|
fsLocal: FakeFs,
|
||||||
|
fsRemote: FakeFs,
|
||||||
|
fsEncrypt: FakeFsEncrypt,
|
||||||
|
profiler: Profiler,
|
||||||
|
db: InternalDBs,
|
||||||
|
triggerSource: SyncTriggerSourceType,
|
||||||
|
profileID: string,
|
||||||
|
vaultRandomID: string,
|
||||||
|
configDir: string,
|
||||||
|
settings: RemotelySavePluginSettings,
|
||||||
|
getProtectModifyPercentageErrorStrFunc: any,
|
||||||
|
markIsSyncingFunc: (isSyncing: boolean) => void,
|
||||||
|
notifyFunc?: (s: SyncTriggerSourceType, step: number) => Promise<any>,
|
||||||
|
errNotifyFunc?: (s: SyncTriggerSourceType, error: Error) => Promise<any>,
|
||||||
|
ribboonFunc?: (s: SyncTriggerSourceType, step: number) => Promise<any>,
|
||||||
|
statusBarFunc?: (s: SyncTriggerSourceType, step: number) => any,
|
||||||
|
callbackSyncProcess?: any
|
||||||
|
) {
|
||||||
|
markIsSyncingFunc(true);
|
||||||
|
|
||||||
|
let step = 0; // dry mode only
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
|
||||||
|
step = 1;
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert("start big sync func");
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (fsEncrypt.innerFs !== fsRemote) {
|
||||||
|
throw Error(`your enc should has inner of the remote`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const passwordCheckResult = await fsEncrypt.isPasswordOk();
|
||||||
|
if (!passwordCheckResult.ok) {
|
||||||
|
throw Error(passwordCheckResult.reason);
|
||||||
|
}
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(
|
||||||
|
`finish step${step} (list partial remote and check password)`
|
||||||
|
);
|
||||||
|
|
||||||
|
step = 2;
|
||||||
|
const remoteEntityList = await fsEncrypt.walk();
|
||||||
|
console.debug(`remoteEntityList:`);
|
||||||
|
console.debug(remoteEntityList);
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(`finish step${step} (list remote)`);
|
||||||
|
|
||||||
|
step = 3;
|
||||||
|
const localEntityList = await fsLocal.walk();
|
||||||
|
console.debug(`localEntityList:`);
|
||||||
|
console.debug(localEntityList);
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(`finish step${step} (list local)`);
|
||||||
|
|
||||||
|
step = 4;
|
||||||
|
const prevSyncEntityList = await getAllPrevSyncRecordsByVaultAndProfile(
|
||||||
|
db,
|
||||||
|
vaultRandomID,
|
||||||
|
profileID
|
||||||
|
);
|
||||||
|
console.debug(`prevSyncEntityList:`);
|
||||||
|
console.debug(prevSyncEntityList);
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(`finish step${step} (prev sync)`);
|
||||||
|
|
||||||
|
step = 5;
|
||||||
|
let mixedEntityMappings = await ensembleMixedEnties(
|
||||||
|
localEntityList,
|
||||||
|
prevSyncEntityList,
|
||||||
|
remoteEntityList,
|
||||||
|
settings.syncConfigDir ?? false,
|
||||||
|
configDir,
|
||||||
|
settings.syncUnderscoreItems ?? false,
|
||||||
|
settings.ignorePaths ?? [],
|
||||||
|
fsEncrypt,
|
||||||
|
settings.serviceType,
|
||||||
|
profiler
|
||||||
|
);
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(`finish step${step} (build partial mixedEntity)`);
|
||||||
|
|
||||||
|
step = 6;
|
||||||
|
mixedEntityMappings = await getSyncPlanInplace(
|
||||||
|
mixedEntityMappings,
|
||||||
|
settings.howToCleanEmptyFolder ?? "skip",
|
||||||
|
settings.skipSizeLargerThan ?? -1,
|
||||||
|
settings.conflictAction ?? "keep_newer",
|
||||||
|
settings.syncDirection ?? "bidirectional",
|
||||||
|
profiler
|
||||||
|
);
|
||||||
|
console.info(`mixedEntityMappings:`);
|
||||||
|
console.info(mixedEntityMappings); // for debugging
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert("finish building full sync plan");
|
||||||
|
|
||||||
|
await insertSyncPlanRecordByVault(
|
||||||
|
db,
|
||||||
|
mixedEntityMappings,
|
||||||
|
vaultRandomID,
|
||||||
|
settings.serviceType
|
||||||
|
);
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert("finish writing sync plan");
|
||||||
|
profiler.insert(`finish step${step} (make plan)`);
|
||||||
|
|
||||||
|
// The operations above are almost read only and kind of safe.
|
||||||
|
// The operations below begins to write or delete (!!!) something.
|
||||||
|
|
||||||
|
step = 7;
|
||||||
|
if (triggerSource !== "dry") {
|
||||||
|
await doActualSync(
|
||||||
|
mixedEntityMappings,
|
||||||
|
fsLocal,
|
||||||
|
fsEncrypt,
|
||||||
|
vaultRandomID,
|
||||||
|
profileID,
|
||||||
|
settings.concurrency ?? 5,
|
||||||
|
settings.protectModifyPercentage ?? 50,
|
||||||
|
getProtectModifyPercentageErrorStrFunc,
|
||||||
|
db,
|
||||||
|
profiler,
|
||||||
|
callbackSyncProcess
|
||||||
|
);
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(`finish step${step} (actual sync)`);
|
||||||
|
} else {
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
profiler.insert(
|
||||||
|
`finish step${step} (skip actual sync because of dry run)`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
profiler.insert("start error branch");
|
||||||
|
await errNotifyFunc?.(triggerSource, error as Error);
|
||||||
|
|
||||||
|
profiler.insert("finish error branch");
|
||||||
|
} finally {
|
||||||
|
}
|
||||||
|
|
||||||
|
profiler.insert("finish syncRun");
|
||||||
|
console.debug(profiler.toString());
|
||||||
|
await profiler.save(db, vaultRandomID, settings.serviceType);
|
||||||
|
|
||||||
|
step = 8;
|
||||||
|
await notifyFunc?.(triggerSource, step);
|
||||||
|
await ribboonFunc?.(triggerSource, step);
|
||||||
|
await statusBarFunc?.(triggerSource, step);
|
||||||
|
|
||||||
|
markIsSyncingFunc(false);
|
||||||
|
}
|
||||||
|
|
|
@ -119,7 +119,7 @@ export class SyncAlgoV3Modal extends Modal {
|
||||||
this.plugin.saveAgreeToUseNewSyncAlgorithm();
|
this.plugin.saveAgreeToUseNewSyncAlgorithm();
|
||||||
this.plugin.enableAutoSyncIfSet();
|
this.plugin.enableAutoSyncIfSet();
|
||||||
this.plugin.enableInitSyncIfSet();
|
this.plugin.enableInitSyncIfSet();
|
||||||
this.plugin.enableSyncOnSaveIfSet();
|
this.plugin.toggleSyncOnSaveIfSet();
|
||||||
} else {
|
} else {
|
||||||
console.info("do not agree to use the new algorithm");
|
console.info("do not agree to use the new algorithm");
|
||||||
this.plugin.unload();
|
this.plugin.unload();
|
||||||
|
|
|
@ -68,6 +68,7 @@ module.exports = {
|
||||||
// crypto: false,
|
// crypto: false,
|
||||||
// domain: require.resolve("domain-browser"),
|
// domain: require.resolve("domain-browser"),
|
||||||
// events: require.resolve("events"),
|
// events: require.resolve("events"),
|
||||||
|
fs: false,
|
||||||
http: false,
|
http: false,
|
||||||
// http: require.resolve("stream-http"),
|
// http: require.resolve("stream-http"),
|
||||||
https: false,
|
https: false,
|
||||||
|
@ -87,6 +88,7 @@ module.exports = {
|
||||||
url: require.resolve("url/"),
|
url: require.resolve("url/"),
|
||||||
// util: require.resolve("util"),
|
// util: require.resolve("util"),
|
||||||
// vm: require.resolve("vm-browserify"),
|
// vm: require.resolve("vm-browserify"),
|
||||||
|
vm: false,
|
||||||
// zlib: require.resolve("browserify-zlib"),
|
// zlib: require.resolve("browserify-zlib"),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
Loading…
Reference in New Issue