mirror of
https://github.com/remotely-save/remotely-save.git
synced 2024-06-07 21:10:45 +00:00
basically workable webdav
This commit is contained in:
fyears
parent
ce0cc232c8
commit
d1839706af
13
src/baseTypes.ts
Normal file
13
src/baseTypes.ts
Normal file
@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Only type defs here.
|
||||
*/
|
||||
|
||||
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav";
|
||||
|
||||
export interface RemoteItem {
|
||||
key: string;
|
||||
lastModified: number;
|
||||
size: number;
|
||||
remoteType: SUPPORTED_SERVICES_TYPE;
|
||||
etag?: string;
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
import localforage from "localforage";
|
||||
import { TAbstractFile, TFile, TFolder } from "obsidian";
|
||||
|
||||
import type { SUPPORTED_SERVICES_TYPE } from "./misc";
|
||||
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
||||
import type { SyncPlanType } from "./sync";
|
||||
|
||||
export type LocalForage = typeof localforage;
|
||||
@ -197,7 +197,8 @@ export const insertRenameRecord = async (
|
||||
await db.deleteHistoryTbl.setItem(k.key, k);
|
||||
};
|
||||
|
||||
export const upsertSyncMetaMappingDataS3 = async (
|
||||
export const upsertSyncMetaMappingData = async (
|
||||
serviceType: SUPPORTED_SERVICES_TYPE,
|
||||
db: InternalDBs,
|
||||
localKey: string,
|
||||
localMTime: number,
|
||||
@ -215,13 +216,14 @@ export const upsertSyncMetaMappingDataS3 = async (
|
||||
remoteMtime: remoteMTime,
|
||||
remoteSize: remoteSize,
|
||||
remoteExtraKey: remoteExtraKey,
|
||||
remoteType: "s3",
|
||||
remoteType: serviceType,
|
||||
keyType: localKey.endsWith("/") ? "folder" : "file",
|
||||
};
|
||||
await db.syncMappingTbl.setItem(remoteKey, aggregratedInfo);
|
||||
};
|
||||
|
||||
export const getSyncMetaMappingByRemoteKeyS3 = async (
|
||||
export const getSyncMetaMappingByRemoteKey = async (
|
||||
serviceType: SUPPORTED_SERVICES_TYPE,
|
||||
db: InternalDBs,
|
||||
remoteKey: string,
|
||||
remoteMTime: number,
|
||||
@ -240,7 +242,7 @@ export const getSyncMetaMappingByRemoteKeyS3 = async (
|
||||
potentialItem.remoteKey === remoteKey &&
|
||||
potentialItem.remoteMtime === remoteMTime &&
|
||||
potentialItem.remoteExtraKey === remoteExtraKey &&
|
||||
potentialItem.remoteType === "s3"
|
||||
potentialItem.remoteType === serviceType
|
||||
) {
|
||||
// the result was found
|
||||
return potentialItem;
|
||||
|
||||
128
src/main.ts
128
src/main.ts
@ -25,22 +25,20 @@ import type { InternalDBs } from "./localdb";
|
||||
|
||||
import type { SyncStatusType, PasswordCheckType } from "./sync";
|
||||
import { isPasswordOk, getSyncPlan, doActualSync } from "./sync";
|
||||
import {
|
||||
DEFAULT_S3_CONFIG,
|
||||
getS3Client,
|
||||
listFromRemote,
|
||||
S3Config,
|
||||
checkS3Connectivity,
|
||||
} from "./s3";
|
||||
import { S3Config, DEFAULT_S3_CONFIG } from "./s3";
|
||||
import { WebdavConfig, DEFAULT_WEBDAV_CONFIG } from "./webdav";
|
||||
import { RemoteClient } from "./remote";
|
||||
import { exportSyncPlansToFiles } from "./debugMode";
|
||||
|
||||
interface RemotelySavePluginSettings {
|
||||
s3?: S3Config;
|
||||
password?: string;
|
||||
s3: S3Config;
|
||||
webdav: WebdavConfig;
|
||||
password: string;
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
|
||||
s3: DEFAULT_S3_CONFIG,
|
||||
webdav: DEFAULT_WEBDAV_CONFIG,
|
||||
password: "",
|
||||
};
|
||||
|
||||
@ -71,6 +69,16 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
})
|
||||
);
|
||||
|
||||
this.addRibbonIcon("dice", "Remotely Save", async () => {
|
||||
const client = new RemoteClient(
|
||||
"webdav",
|
||||
undefined,
|
||||
this.settings.webdav
|
||||
);
|
||||
const xx = await client.listFromRemote()
|
||||
console.log(xx);
|
||||
});
|
||||
|
||||
this.addRibbonIcon("switch", "Remotely Save", async () => {
|
||||
if (this.syncStatus !== "idle") {
|
||||
new Notice(
|
||||
@ -86,8 +94,13 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
|
||||
new Notice("2/6 Starting to fetch remote meta data.");
|
||||
this.syncStatus = "getting_remote_meta";
|
||||
const s3Client = getS3Client(this.settings.s3);
|
||||
const remoteRsp = await listFromRemote(s3Client, this.settings.s3);
|
||||
// const client = new RemoteClient('s3', this.settings.s3, undefined);
|
||||
const client = new RemoteClient(
|
||||
"webdav",
|
||||
undefined,
|
||||
this.settings.webdav
|
||||
);
|
||||
const remoteRsp = await client.listFromRemote();
|
||||
|
||||
new Notice("3/6 Starting to fetch local meta data.");
|
||||
this.syncStatus = "getting_local_meta";
|
||||
@ -115,6 +128,7 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
local,
|
||||
localHistory,
|
||||
this.db,
|
||||
client.serviceType,
|
||||
this.settings.password
|
||||
);
|
||||
console.log(syncPlan.mixedStates); // for debugging
|
||||
@ -127,8 +141,7 @@ export default class RemotelySavePlugin extends Plugin {
|
||||
|
||||
this.syncStatus = "syncing";
|
||||
await doActualSync(
|
||||
s3Client,
|
||||
this.settings.s3,
|
||||
client,
|
||||
this.db,
|
||||
this.app.vault,
|
||||
syncPlan,
|
||||
@ -259,6 +272,86 @@ class RemotelySaveSettingTab extends PluginSettingTab {
|
||||
|
||||
containerEl.createEl("h1", { text: "Remotely Save" });
|
||||
|
||||
const webdavDiv = containerEl.createEl("div");
|
||||
webdavDiv.createEl("h2", { text: "Webdav Service" });
|
||||
new Setting(webdavDiv)
|
||||
.setName("server address")
|
||||
.setDesc("server address")
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(this.plugin.settings.webdav.address)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.webdav.address = value.trim();
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(webdavDiv)
|
||||
.setName("server username")
|
||||
.setDesc("server username")
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(this.plugin.settings.webdav.username)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.webdav.username = value.trim();
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(webdavDiv)
|
||||
.setName("server password")
|
||||
.setDesc("server password")
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(this.plugin.settings.webdav.password)
|
||||
.onChange(async (value) => {
|
||||
this.plugin.settings.webdav.password = value.trim();
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(webdavDiv)
|
||||
.setName("server auth type")
|
||||
.setDesc("server auth type")
|
||||
.addText((text) =>
|
||||
text
|
||||
.setPlaceholder("")
|
||||
.setValue(this.plugin.settings.webdav.authType)
|
||||
.onChange(async (value) => {
|
||||
if (value.trim() === "digest") {
|
||||
this.plugin.settings.webdav.authType = "digest";
|
||||
} else {
|
||||
this.plugin.settings.webdav.authType = "basic";
|
||||
}
|
||||
|
||||
await this.plugin.saveSettings();
|
||||
})
|
||||
);
|
||||
|
||||
new Setting(webdavDiv)
|
||||
.setName("check connectivity")
|
||||
.setDesc("check connectivity")
|
||||
.addButton(async (button) => {
|
||||
button.setButtonText("Check");
|
||||
button.onClick(async () => {
|
||||
new Notice("Checking...");
|
||||
const client = new RemoteClient(
|
||||
"webdav",
|
||||
undefined,
|
||||
this.plugin.settings.webdav
|
||||
);
|
||||
const res = await client.checkConnectivity();
|
||||
if (res) {
|
||||
new Notice("Great! The webdav server can be accessed.");
|
||||
} else {
|
||||
new Notice("The webdav server cannot be reached.");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const s3Div = containerEl.createEl("div");
|
||||
s3Div.createEl("h2", { text: "S3 (-compatible) Service" });
|
||||
|
||||
@ -368,11 +461,12 @@ class RemotelySaveSettingTab extends PluginSettingTab {
|
||||
button.setButtonText("Check");
|
||||
button.onClick(async () => {
|
||||
new Notice("Checking...");
|
||||
const s3Client = getS3Client(this.plugin.settings.s3);
|
||||
const res = await checkS3Connectivity(
|
||||
s3Client,
|
||||
this.plugin.settings.s3
|
||||
const client = new RemoteClient(
|
||||
"s3",
|
||||
this.plugin.settings.s3,
|
||||
undefined
|
||||
);
|
||||
const res = await client.checkConnectivity();
|
||||
if (res) {
|
||||
new Notice("Great! The bucket can be accessed.");
|
||||
} else {
|
||||
|
||||
16
src/misc.ts
16
src/misc.ts
@ -4,8 +4,6 @@ import * as path from "path";
|
||||
import { base32 } from "rfc4648";
|
||||
import XRegExp from "xregexp";
|
||||
|
||||
export type SUPPORTED_SERVICES_TYPE = "s3" | "webdav" | "ftp";
|
||||
|
||||
/**
|
||||
* If any part of the file starts with '.' or '_' then it's a hidden file.
|
||||
* @param item
|
||||
@ -130,3 +128,17 @@ export const isVaildText = (a: string) => {
|
||||
a
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* If input is already a folder, returns it as is;
|
||||
* And if input is a file, returns its direname.
|
||||
* @param a
|
||||
* @returns
|
||||
*/
|
||||
export const getPathFolder = (a: string) => {
|
||||
if (a.endsWith("/")) {
|
||||
return a;
|
||||
}
|
||||
const b = path.posix.dirname(a);
|
||||
return b.endsWith("/") ? b : `${b}/`;
|
||||
};
|
||||
|
||||
151
src/remote.ts
Normal file
151
src/remote.ts
Normal file
@ -0,0 +1,151 @@
|
||||
import { Vault } from "obsidian";
|
||||
|
||||
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
||||
import * as s3 from "./s3";
|
||||
import * as webdav from "./webdav";
|
||||
|
||||
export class RemoteClient {
|
||||
readonly serviceType: SUPPORTED_SERVICES_TYPE;
|
||||
readonly s3Client?: s3.S3Client;
|
||||
readonly s3Config?: s3.S3Config;
|
||||
readonly webdavClient?: webdav.WebDAVClient;
|
||||
readonly webdavConfig?: webdav.WebdavConfig;
|
||||
|
||||
constructor(
|
||||
serviceType: SUPPORTED_SERVICES_TYPE,
|
||||
s3Config?: s3.S3Config,
|
||||
webdavConfig?: webdav.WebdavConfig
|
||||
) {
|
||||
this.serviceType = serviceType;
|
||||
if (serviceType === "s3") {
|
||||
this.s3Config = s3Config;
|
||||
this.s3Client = s3.getS3Client(s3Config);
|
||||
} else if (serviceType === "webdav") {
|
||||
this.webdavConfig = webdavConfig;
|
||||
this.webdavClient = webdav.getWebdavClient(webdavConfig);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
}
|
||||
|
||||
getRemoteMeta = async (fileOrFolderPath: string) => {
|
||||
if (this.serviceType === "s3") {
|
||||
return await s3.getRemoteMeta(
|
||||
this.s3Client,
|
||||
this.s3Config,
|
||||
fileOrFolderPath
|
||||
);
|
||||
} else if (this.serviceType === "webdav") {
|
||||
return await webdav.getRemoteMeta(this.webdavClient, fileOrFolderPath);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
};
|
||||
|
||||
uploadToRemote = async (
|
||||
fileOrFolderPath: string,
|
||||
vault: Vault,
|
||||
isRecursively: boolean = false,
|
||||
password: string = "",
|
||||
remoteEncryptedKey: string = ""
|
||||
) => {
|
||||
if (this.serviceType === "s3") {
|
||||
return await s3.uploadToRemote(
|
||||
this.s3Client,
|
||||
this.s3Config,
|
||||
fileOrFolderPath,
|
||||
vault,
|
||||
isRecursively,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else if (this.serviceType === "webdav") {
|
||||
return await webdav.uploadToRemote(
|
||||
this.webdavClient,
|
||||
fileOrFolderPath,
|
||||
vault,
|
||||
isRecursively,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
};
|
||||
|
||||
listFromRemote = async (prefix?: string) => {
|
||||
if (this.serviceType === "s3") {
|
||||
return await s3.listFromRemote(this.s3Client, this.s3Config, prefix);
|
||||
} else if (this.serviceType === "webdav") {
|
||||
return await webdav.listFromRemote(this.webdavClient, prefix);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
};
|
||||
|
||||
downloadFromRemote = async (
|
||||
fileOrFolderPath: string,
|
||||
vault: Vault,
|
||||
mtime: number,
|
||||
password: string = "",
|
||||
remoteEncryptedKey: string = ""
|
||||
) => {
|
||||
if (this.serviceType === "s3") {
|
||||
return await s3.downloadFromRemote(
|
||||
this.s3Client,
|
||||
this.s3Config,
|
||||
fileOrFolderPath,
|
||||
vault,
|
||||
mtime,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else if (this.serviceType === "webdav") {
|
||||
return await webdav.downloadFromRemote(
|
||||
this.webdavClient,
|
||||
fileOrFolderPath,
|
||||
vault,
|
||||
mtime,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
};
|
||||
|
||||
deleteFromRemote = async (
|
||||
fileOrFolderPath: string,
|
||||
password: string = "",
|
||||
remoteEncryptedKey: string = ""
|
||||
) => {
|
||||
if (this.serviceType === "s3") {
|
||||
return await s3.deleteFromRemote(
|
||||
this.s3Client,
|
||||
this.s3Config,
|
||||
fileOrFolderPath,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else if (this.serviceType === "webdav") {
|
||||
return await webdav.deleteFromRemote(
|
||||
this.webdavClient,
|
||||
fileOrFolderPath,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
};
|
||||
|
||||
checkConnectivity = async () => {
|
||||
if (this.serviceType === "s3") {
|
||||
return await s3.checkConnectivity(this.s3Client, this.s3Config);
|
||||
} else if (this.serviceType === "webdav") {
|
||||
return await webdav.checkConnectivity(this.webdavClient);
|
||||
} else {
|
||||
throw Error(`not supported service type ${this.serviceType}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
42
src/s3.ts
42
src/s3.ts
@ -14,7 +14,9 @@ import {
|
||||
HeadBucketCommand,
|
||||
ListObjectsV2CommandInput,
|
||||
ListObjectsV2CommandOutput,
|
||||
HeadObjectCommandOutput,
|
||||
} from "@aws-sdk/client-s3";
|
||||
export { S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
import type { _Object } from "@aws-sdk/client-s3";
|
||||
|
||||
@ -24,6 +26,8 @@ import {
|
||||
mkdirpInVault,
|
||||
} from "./misc";
|
||||
import * as mime from "mime-types";
|
||||
|
||||
import { RemoteItem } from "./baseTypes";
|
||||
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
||||
|
||||
export interface S3Config {
|
||||
@ -44,6 +48,29 @@ export const DEFAULT_S3_CONFIG = {
|
||||
|
||||
export type S3ObjectType = _Object;
|
||||
|
||||
const fromS3ObjectToRemoteItem = (x: S3ObjectType) => {
|
||||
return {
|
||||
key: x.Key,
|
||||
lastModified: x.LastModified.valueOf(),
|
||||
size: x.Size,
|
||||
remoteType: "s3",
|
||||
etag: x.ETag,
|
||||
} as RemoteItem;
|
||||
};
|
||||
|
||||
const fromS3HeadObjectToRemoteItem = (
|
||||
key: string,
|
||||
x: HeadObjectCommandOutput
|
||||
) => {
|
||||
return {
|
||||
key: key,
|
||||
lastModified: x.LastModified.valueOf(),
|
||||
size: x.ContentLength,
|
||||
remoteType: "s3",
|
||||
etag: x.ETag,
|
||||
} as RemoteItem;
|
||||
};
|
||||
|
||||
export const getS3Client = (s3Config: S3Config) => {
|
||||
let endpoint = s3Config.s3Endpoint;
|
||||
if (!(endpoint.startsWith("http://") || endpoint.startsWith("https://"))) {
|
||||
@ -65,12 +92,14 @@ export const getRemoteMeta = async (
|
||||
s3Config: S3Config,
|
||||
fileOrFolderPath: string
|
||||
) => {
|
||||
return await s3Client.send(
|
||||
const res = await s3Client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: fileOrFolderPath,
|
||||
})
|
||||
);
|
||||
|
||||
return fromS3HeadObjectToRemoteItem(fileOrFolderPath, res);
|
||||
};
|
||||
|
||||
export const uploadToRemote = async (
|
||||
@ -181,10 +210,7 @@ export const listFromRemote = async (
|
||||
|
||||
// ensemble fake rsp
|
||||
return {
|
||||
"$.metadata": {
|
||||
httpStatusCode: 200,
|
||||
},
|
||||
Contents: contents,
|
||||
Contents: contents.map((x) => fromS3ObjectToRemoteItem(x)),
|
||||
};
|
||||
};
|
||||
|
||||
@ -214,7 +240,7 @@ const getObjectBodyToArrayBuffer = async (
|
||||
}
|
||||
};
|
||||
|
||||
export const downloadFromRemoteRaw = async (
|
||||
const downloadFromRemoteRaw = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
fileOrFolderPath: string
|
||||
@ -302,7 +328,7 @@ export const deleteFromRemote = async (
|
||||
await s3Client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: s3Config.s3BucketName,
|
||||
Key: element.Key,
|
||||
Key: element.key,
|
||||
})
|
||||
);
|
||||
});
|
||||
@ -320,7 +346,7 @@ export const deleteFromRemote = async (
|
||||
* @param s3Config
|
||||
* @returns
|
||||
*/
|
||||
export const checkS3Connectivity = async (
|
||||
export const checkConnectivity = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config
|
||||
) => {
|
||||
|
||||
249
src/sync.ts
249
src/sync.ts
@ -1,27 +1,15 @@
|
||||
import { TAbstractFile, TFolder, TFile, Vault } from "obsidian";
|
||||
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
|
||||
import {
|
||||
clearDeleteRenameHistoryOfKey,
|
||||
upsertSyncMetaMappingDataS3,
|
||||
getSyncMetaMappingByRemoteKeyS3,
|
||||
upsertSyncMetaMappingData,
|
||||
getSyncMetaMappingByRemoteKey,
|
||||
} from "./localdb";
|
||||
import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
|
||||
|
||||
import {
|
||||
S3Config,
|
||||
S3ObjectType,
|
||||
uploadToRemote,
|
||||
deleteFromRemote,
|
||||
downloadFromRemote,
|
||||
} from "./s3";
|
||||
import {
|
||||
mkdirpInVault,
|
||||
SUPPORTED_SERVICES_TYPE,
|
||||
isHiddenPath,
|
||||
isVaildText,
|
||||
} from "./misc";
|
||||
import { RemoteClient } from "./remote";
|
||||
import type { SUPPORTED_SERVICES_TYPE, RemoteItem } from "./baseTypes";
|
||||
import { mkdirpInVault, isHiddenPath, isVaildText } from "./misc";
|
||||
import {
|
||||
decryptBase32ToString,
|
||||
encryptStringToBase32,
|
||||
@ -85,7 +73,7 @@ export interface PasswordCheckType {
|
||||
}
|
||||
|
||||
export const isPasswordOk = async (
|
||||
remote: S3ObjectType[],
|
||||
remote: RemoteItem[],
|
||||
password: string = ""
|
||||
) => {
|
||||
if (remote === undefined || remote.length === 0) {
|
||||
@ -95,7 +83,7 @@ export const isPasswordOk = async (
|
||||
reason: "empty_remote",
|
||||
} as PasswordCheckType;
|
||||
}
|
||||
const santyCheckKey = remote[0].Key;
|
||||
const santyCheckKey = remote[0].key;
|
||||
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
||||
// this is encrypted!
|
||||
// try to decrypt it using the provided password.
|
||||
@ -143,26 +131,28 @@ export const isPasswordOk = async (
|
||||
};
|
||||
|
||||
const ensembleMixedStates = async (
|
||||
remote: S3ObjectType[],
|
||||
remote: RemoteItem[],
|
||||
local: TAbstractFile[],
|
||||
deleteHistory: FileFolderHistoryRecord[],
|
||||
db: InternalDBs,
|
||||
remoteType: SUPPORTED_SERVICES_TYPE,
|
||||
password: string = ""
|
||||
) => {
|
||||
const results = {} as Record<string, FileOrFolderMixedState>;
|
||||
|
||||
if (remote !== undefined) {
|
||||
for (const entry of remote) {
|
||||
const remoteEncryptedKey = entry.Key;
|
||||
const remoteEncryptedKey = entry.key;
|
||||
let key = remoteEncryptedKey;
|
||||
if (password !== "") {
|
||||
key = await decryptBase32ToString(remoteEncryptedKey, password);
|
||||
}
|
||||
const backwardMapping = await getSyncMetaMappingByRemoteKeyS3(
|
||||
const backwardMapping = await getSyncMetaMappingByRemoteKey(
|
||||
remoteType,
|
||||
db,
|
||||
key,
|
||||
entry.LastModified.valueOf(),
|
||||
entry.ETag
|
||||
entry.lastModified,
|
||||
entry.etag
|
||||
);
|
||||
|
||||
let r = {} as FileOrFolderMixedState;
|
||||
@ -179,8 +169,8 @@ const ensembleMixedStates = async (
|
||||
r = {
|
||||
key: key,
|
||||
exist_remote: true,
|
||||
mtime_remote: entry.LastModified.valueOf(),
|
||||
size_remote: entry.Size,
|
||||
mtime_remote: entry.lastModified,
|
||||
size_remote: entry.size,
|
||||
remote_encrypted_key: remoteEncryptedKey,
|
||||
};
|
||||
}
|
||||
@ -402,10 +392,11 @@ const getOperation = (
|
||||
};
|
||||
|
||||
export const getSyncPlan = async (
|
||||
remote: S3ObjectType[],
|
||||
remote: RemoteItem[],
|
||||
local: TAbstractFile[],
|
||||
deleteHistory: FileFolderHistoryRecord[],
|
||||
db: InternalDBs,
|
||||
remoteType: SUPPORTED_SERVICES_TYPE,
|
||||
password: string = ""
|
||||
) => {
|
||||
const mixedStates = await ensembleMixedStates(
|
||||
@ -413,6 +404,7 @@ export const getSyncPlan = async (
|
||||
local,
|
||||
deleteHistory,
|
||||
db,
|
||||
remoteType,
|
||||
password
|
||||
);
|
||||
for (const [key, val] of Object.entries(mixedStates)) {
|
||||
@ -420,15 +412,105 @@ export const getSyncPlan = async (
|
||||
}
|
||||
const plan = {
|
||||
ts: Date.now(),
|
||||
remoteType: "s3",
|
||||
remoteType: remoteType,
|
||||
mixedStates: mixedStates,
|
||||
} as SyncPlanType;
|
||||
return plan;
|
||||
};
|
||||
|
||||
const dispatchOperationToActual = async (
|
||||
key: string,
|
||||
state: FileOrFolderMixedState,
|
||||
client: RemoteClient,
|
||||
db: InternalDBs,
|
||||
vault: Vault,
|
||||
password: string = ""
|
||||
) => {
|
||||
let remoteEncryptedKey = key;
|
||||
if (password !== "") {
|
||||
remoteEncryptedKey = state.remote_encrypted_key;
|
||||
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
||||
remoteEncryptedKey = await encryptStringToBase32(key, password);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
state.decision === undefined ||
|
||||
state.decision === "unknown" ||
|
||||
state.decision === "undecided"
|
||||
) {
|
||||
throw Error(`unknown decision in ${JSON.stringify(state)}`);
|
||||
} else if (state.decision === "skip") {
|
||||
// do nothing
|
||||
} else if (state.decision === "download_clearhist") {
|
||||
await client.downloadFromRemote(
|
||||
state.key,
|
||||
vault,
|
||||
state.mtime_remote,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "upload_clearhist") {
|
||||
const remoteObjMeta = await client.uploadToRemote(
|
||||
state.key,
|
||||
vault,
|
||||
false,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await upsertSyncMetaMappingData(
|
||||
client.serviceType,
|
||||
db,
|
||||
state.key,
|
||||
state.mtime_local,
|
||||
state.size_local,
|
||||
state.key,
|
||||
remoteObjMeta.lastModified,
|
||||
remoteObjMeta.size,
|
||||
remoteObjMeta.etag
|
||||
);
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "download") {
|
||||
await mkdirpInVault(state.key, vault);
|
||||
await client.downloadFromRemote(
|
||||
state.key,
|
||||
vault,
|
||||
state.mtime_remote,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else if (state.decision === "delremote_clearhist") {
|
||||
await client.deleteFromRemote(state.key, password, remoteEncryptedKey);
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "upload") {
|
||||
const remoteObjMeta = await client.uploadToRemote(
|
||||
state.key,
|
||||
vault,
|
||||
false,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await upsertSyncMetaMappingData(
|
||||
client.serviceType,
|
||||
db,
|
||||
state.key,
|
||||
state.mtime_local,
|
||||
state.size_local,
|
||||
state.key,
|
||||
remoteObjMeta.lastModified,
|
||||
remoteObjMeta.size,
|
||||
remoteObjMeta.etag
|
||||
);
|
||||
} else if (state.decision === "clearhist") {
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else {
|
||||
throw Error("this should never happen!");
|
||||
}
|
||||
};
|
||||
|
||||
export const doActualSync = async (
|
||||
s3Client: S3Client,
|
||||
s3Config: S3Config,
|
||||
client: RemoteClient,
|
||||
db: InternalDBs,
|
||||
vault: Vault,
|
||||
syncPlan: SyncPlanType,
|
||||
@ -438,102 +520,15 @@ export const doActualSync = async (
|
||||
await Promise.all(
|
||||
Object.entries(keyStates)
|
||||
.sort((k, v) => -(k as string).length)
|
||||
.map(async ([k, v]) => {
|
||||
const key = k as string;
|
||||
const state = v as FileOrFolderMixedState;
|
||||
let remoteEncryptedKey = key;
|
||||
if (password !== "") {
|
||||
remoteEncryptedKey = state.remote_encrypted_key;
|
||||
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
||||
remoteEncryptedKey = await encryptStringToBase32(key, password);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
state.decision === undefined ||
|
||||
state.decision === "unknown" ||
|
||||
state.decision === "undecided"
|
||||
) {
|
||||
throw Error(`unknown decision in ${JSON.stringify(state)}`);
|
||||
} else if (state.decision === "skip") {
|
||||
// do nothing
|
||||
} else if (state.decision === "download_clearhist") {
|
||||
await downloadFromRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
vault,
|
||||
state.mtime_remote,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "upload_clearhist") {
|
||||
const remoteObjMeta = await uploadToRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
vault,
|
||||
false,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await upsertSyncMetaMappingDataS3(
|
||||
db,
|
||||
state.key,
|
||||
state.mtime_local,
|
||||
state.size_local,
|
||||
state.key,
|
||||
remoteObjMeta.LastModified.valueOf(),
|
||||
remoteObjMeta.ContentLength,
|
||||
remoteObjMeta.ETag
|
||||
);
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "download") {
|
||||
await mkdirpInVault(state.key, vault);
|
||||
await downloadFromRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
vault,
|
||||
state.mtime_remote,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
} else if (state.decision === "delremote_clearhist") {
|
||||
await deleteFromRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else if (state.decision === "upload") {
|
||||
const remoteObjMeta = await uploadToRemote(
|
||||
s3Client,
|
||||
s3Config,
|
||||
state.key,
|
||||
vault,
|
||||
false,
|
||||
password,
|
||||
remoteEncryptedKey
|
||||
);
|
||||
await upsertSyncMetaMappingDataS3(
|
||||
db,
|
||||
state.key,
|
||||
state.mtime_local,
|
||||
state.size_local,
|
||||
state.key,
|
||||
remoteObjMeta.LastModified.valueOf(),
|
||||
remoteObjMeta.ContentLength,
|
||||
remoteObjMeta.ETag
|
||||
);
|
||||
} else if (state.decision === "clearhist") {
|
||||
await clearDeleteRenameHistoryOfKey(db, state.key);
|
||||
} else {
|
||||
throw Error("this should never happen!");
|
||||
}
|
||||
})
|
||||
.map(async ([k, v]) =>
|
||||
dispatchOperationToActual(
|
||||
k as string,
|
||||
v as FileOrFolderMixedState,
|
||||
client,
|
||||
db,
|
||||
vault,
|
||||
password
|
||||
)
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
@ -2,14 +2,17 @@ import { Buffer } from "buffer";
|
||||
import { FileStats, Vault } from "obsidian";
|
||||
import { AuthType, BufferLike, createClient } from "webdav/web";
|
||||
import type { WebDAVClient, ResponseDataDetailed, FileStat } from "webdav/web";
|
||||
export type { WebDAVClient } from "webdav/web";
|
||||
|
||||
import type { RemoteItem } from "./baseTypes";
|
||||
|
||||
import {
|
||||
arrayBufferToBuffer,
|
||||
bufferToArrayBuffer,
|
||||
mkdirpInVault,
|
||||
getPathFolder,
|
||||
} from "./misc";
|
||||
import { decryptArrayBuffer, encryptArrayBuffer } from "./encrypt";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
export interface WebdavConfig {
|
||||
address: string;
|
||||
@ -25,6 +28,34 @@ export const DEFAULT_WEBDAV_CONFIG = {
|
||||
authType: "basic",
|
||||
} as WebdavConfig;
|
||||
|
||||
const getWebdavPath = (fileOrFolderPath: string) => {
|
||||
if (!fileOrFolderPath.startsWith("/")) {
|
||||
return `/${fileOrFolderPath}`;
|
||||
}
|
||||
return fileOrFolderPath;
|
||||
};
|
||||
|
||||
const getNormPath = (fileOrFolderPath: string) => {
|
||||
if (fileOrFolderPath.startsWith("/")) {
|
||||
return fileOrFolderPath.slice(1);
|
||||
}
|
||||
return fileOrFolderPath;
|
||||
};
|
||||
|
||||
const fromWebdavItemToRemoteItem = (x: FileStat) => {
|
||||
let key = getNormPath(x.filename);
|
||||
if (x.type === "directory" && !key.endsWith("/")) {
|
||||
key = `${key}/`;
|
||||
}
|
||||
return {
|
||||
key: key,
|
||||
lastModified: Date.parse(x.lastmod).valueOf(),
|
||||
size: x.size,
|
||||
remoteType: "webdav",
|
||||
etag: x.etag || undefined,
|
||||
} as RemoteItem;
|
||||
};
|
||||
|
||||
export const getWebdavClient = (webdavConfig: WebdavConfig) => {
|
||||
if (webdavConfig.username !== "" && webdavConfig.password !== "") {
|
||||
return createClient(webdavConfig.address, {
|
||||
@ -41,29 +72,14 @@ export const getWebdavClient = (webdavConfig: WebdavConfig) => {
|
||||
}
|
||||
};
|
||||
|
||||
const getWebdavPath = (fileOrFolderPath: string) => {
|
||||
if (!fileOrFolderPath.startsWith("/")) {
|
||||
return `/${fileOrFolderPath}`;
|
||||
}
|
||||
return fileOrFolderPath;
|
||||
};
|
||||
|
||||
const getNormPath = (fileOrFolderPath: string) => {
|
||||
if (fileOrFolderPath.startsWith("/")) {
|
||||
return fileOrFolderPath.slice(1);
|
||||
}
|
||||
return fileOrFolderPath;
|
||||
};
|
||||
|
||||
export const getRemoteMeta = async (
|
||||
client: WebDAVClient,
|
||||
fileOrFolderPath: string
|
||||
) => {
|
||||
const res = (await client.stat(getWebdavPath(fileOrFolderPath), {
|
||||
details: true,
|
||||
})) as ResponseDataDetailed<FileStat>;
|
||||
res.data.filename = getNormPath(res.data.filename);
|
||||
return res;
|
||||
details: false,
|
||||
})) as FileStat;
|
||||
return fromWebdavItemToRemoteItem(res);
|
||||
};
|
||||
|
||||
export const uploadToRemote = async (
|
||||
@ -88,10 +104,11 @@ export const uploadToRemote = async (
|
||||
// folder
|
||||
if (password === "") {
|
||||
// if not encrypted, mkdir a remote folder
|
||||
client.createDirectory(uploadFile, {
|
||||
await client.createDirectory(uploadFile, {
|
||||
recursive: true,
|
||||
});
|
||||
return await getRemoteMeta(client, uploadFile);
|
||||
const res = await getRemoteMeta(client, uploadFile);
|
||||
return res;
|
||||
} else {
|
||||
// if encrypted, upload a fake file with the encrypted file name
|
||||
await client.putFileContents(uploadFile, "", {
|
||||
@ -111,6 +128,11 @@ export const uploadToRemote = async (
|
||||
if (password !== "") {
|
||||
remoteContent = await encryptArrayBuffer(localContent, password);
|
||||
}
|
||||
// we need to create folders before uploading
|
||||
const dir = getPathFolder(uploadFile);
|
||||
if (dir !== "/" && dir !== "") {
|
||||
await client.createDirectory(dir, { recursive: true });
|
||||
}
|
||||
await client.putFileContents(uploadFile, remoteContent, {
|
||||
overwrite: true,
|
||||
onUploadProgress: (progress) => {
|
||||
@ -131,15 +153,12 @@ export const listFromRemote = async (client: WebDAVClient, prefix?: string) => {
|
||||
details: false /* no need for verbose details here */,
|
||||
glob: "/**" /* avoid dot files by using glob */,
|
||||
})) as FileStat[];
|
||||
for (const singleItem of contents) {
|
||||
singleItem.filename = getNormPath(singleItem.filename);
|
||||
}
|
||||
return {
|
||||
Contents: contents,
|
||||
Contents: contents.map((x) => fromWebdavItemToRemoteItem(x)),
|
||||
};
|
||||
};
|
||||
|
||||
export const downloadFromRemoteRaw = async (
|
||||
const downloadFromRemoteRaw = async (
|
||||
client: WebDAVClient,
|
||||
fileOrFolderPath: string
|
||||
) => {
|
||||
@ -212,15 +231,10 @@ export const deleteFromRemote = async (
|
||||
}
|
||||
};
|
||||
|
||||
export const checkWebdavConnectivity = async (client: WebDAVClient) => {
|
||||
export const checkConnectivity = async (client: WebDAVClient) => {
|
||||
try {
|
||||
const results = await getRemoteMeta(client, "/");
|
||||
if (
|
||||
results === undefined ||
|
||||
results.data === undefined ||
|
||||
results.data.type === undefined ||
|
||||
results.data.type !== "directory"
|
||||
) {
|
||||
if (results === undefined) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@ -89,3 +89,25 @@ describe("Misc: vaild file name tests", () => {
|
||||
expect(x).to.be.true;
|
||||
});
|
||||
});
|
||||
|
||||
describe("Misc: get dirname", () => {
|
||||
it("should return itself for folder", async () => {
|
||||
const x = misc.getPathFolder("ssss/");
|
||||
// console.log(x)
|
||||
expect(x).to.equal("ssss/");
|
||||
});
|
||||
|
||||
it("should return folder for file", async () => {
|
||||
const x = misc.getPathFolder("sss/yyy");
|
||||
// console.log(x)
|
||||
expect(x).to.equal("sss/");
|
||||
});
|
||||
|
||||
it("should treat / specially", async () => {
|
||||
const x = misc.getPathFolder("/");
|
||||
expect(x).to.equal("/");
|
||||
|
||||
const y = misc.getPathFolder("/abc");
|
||||
expect(y).to.equal("/");
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user