602 lines
15 KiB
TypeScript
602 lines
15 KiB
TypeScript
import { TAbstractFile, TFile, TFolder, Vault } from "obsidian";
|
|
import type { RemoteItem, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
|
|
import {
|
|
decryptBase32ToString,
|
|
decryptBase64urlToString,
|
|
encryptStringToBase64url,
|
|
MAGIC_ENCRYPTED_PREFIX_BASE32,
|
|
MAGIC_ENCRYPTED_PREFIX_BASE64URL,
|
|
} from "./encrypt";
|
|
import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
|
|
import {
|
|
clearDeleteRenameHistoryOfKey,
|
|
getSyncMetaMappingByRemoteKey,
|
|
upsertSyncMetaMappingData,
|
|
} from "./localdb";
|
|
import { isHiddenPath, isVaildText, mkdirpInVault } from "./misc";
|
|
import { RemoteClient } from "./remote";
|
|
|
|
export type SyncStatusType =
|
|
| "idle"
|
|
| "preparing"
|
|
| "getting_remote_meta"
|
|
| "getting_local_meta"
|
|
| "checking_password"
|
|
| "generating_plan"
|
|
| "syncing"
|
|
| "finish";
|
|
|
|
type DecisionType =
|
|
| "undecided"
|
|
| "unknown"
|
|
| "upload_clearhist"
|
|
| "download_clearhist"
|
|
| "delremote_clearhist"
|
|
| "download"
|
|
| "upload"
|
|
| "clearhist"
|
|
| "mkdirplocal"
|
|
| "skip";
|
|
|
|
interface FileOrFolderMixedState {
|
|
key: string;
|
|
exist_local?: boolean;
|
|
exist_remote?: boolean;
|
|
mtime_local?: number;
|
|
mtime_remote?: number;
|
|
delete_time_local?: number;
|
|
size_local?: number;
|
|
size_remote?: number;
|
|
decision?: DecisionType;
|
|
syncDone?: "done";
|
|
decision_branch?: number;
|
|
remote_encrypted_key?: string;
|
|
}
|
|
|
|
export interface SyncPlanType {
|
|
ts: number;
|
|
remoteType: SUPPORTED_SERVICES_TYPE;
|
|
mixedStates: Record<string, FileOrFolderMixedState>;
|
|
}
|
|
|
|
export interface PasswordCheckType {
|
|
ok: boolean;
|
|
reason:
|
|
| "ok"
|
|
| "empty_remote"
|
|
| "remote_encrypted_local_no_password"
|
|
| "password_matched"
|
|
| "password_not_matched"
|
|
| "invalid_text_after_decryption"
|
|
| "remote_not_encrypted_local_has_password"
|
|
| "no_password_both_sides";
|
|
}
|
|
|
|
export const isPasswordOk = async (
|
|
remote: RemoteItem[],
|
|
password: string = ""
|
|
) => {
|
|
if (remote === undefined || remote.length === 0) {
|
|
// remote empty
|
|
return {
|
|
ok: true,
|
|
reason: "empty_remote",
|
|
} as PasswordCheckType;
|
|
}
|
|
const santyCheckKey = remote[0].key;
|
|
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
|
// this is encrypted using old base32!
|
|
// try to decrypt it using the provided password.
|
|
if (password === "") {
|
|
return {
|
|
ok: false,
|
|
reason: "remote_encrypted_local_no_password",
|
|
} as PasswordCheckType;
|
|
}
|
|
try {
|
|
const res = await decryptBase32ToString(santyCheckKey, password);
|
|
|
|
// additional test
|
|
// because iOS Safari bypasses decryption with wrong password!
|
|
if (isVaildText(res)) {
|
|
return {
|
|
ok: true,
|
|
reason: "password_matched",
|
|
} as PasswordCheckType;
|
|
} else {
|
|
return {
|
|
ok: false,
|
|
reason: "invalid_text_after_decryption",
|
|
} as PasswordCheckType;
|
|
}
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
reason: "password_not_matched",
|
|
} as PasswordCheckType;
|
|
}
|
|
}
|
|
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
|
|
// this is encrypted using new base64url!
|
|
// try to decrypt it using the provided password.
|
|
if (password === "") {
|
|
return {
|
|
ok: false,
|
|
reason: "remote_encrypted_local_no_password",
|
|
} as PasswordCheckType;
|
|
}
|
|
try {
|
|
const res = await decryptBase64urlToString(santyCheckKey, password);
|
|
|
|
// additional test
|
|
// because iOS Safari bypasses decryption with wrong password!
|
|
if (isVaildText(res)) {
|
|
return {
|
|
ok: true,
|
|
reason: "password_matched",
|
|
} as PasswordCheckType;
|
|
} else {
|
|
return {
|
|
ok: false,
|
|
reason: "invalid_text_after_decryption",
|
|
} as PasswordCheckType;
|
|
}
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
reason: "password_not_matched",
|
|
} as PasswordCheckType;
|
|
}
|
|
} else {
|
|
// it is not encrypted!
|
|
if (password !== "") {
|
|
return {
|
|
ok: false,
|
|
reason: "remote_not_encrypted_local_has_password",
|
|
} as PasswordCheckType;
|
|
}
|
|
return {
|
|
ok: true,
|
|
reason: "no_password_both_sides",
|
|
} as PasswordCheckType;
|
|
}
|
|
};
|
|
|
|
const ensembleMixedStates = async (
|
|
remote: RemoteItem[],
|
|
local: TAbstractFile[],
|
|
deleteHistory: FileFolderHistoryRecord[],
|
|
db: InternalDBs,
|
|
remoteType: SUPPORTED_SERVICES_TYPE,
|
|
password: string = ""
|
|
) => {
|
|
const results = {} as Record<string, FileOrFolderMixedState>;
|
|
|
|
if (remote !== undefined) {
|
|
for (const entry of remote) {
|
|
const remoteEncryptedKey = entry.key;
|
|
let key = remoteEncryptedKey;
|
|
if (password !== "") {
|
|
if (remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
|
key = await decryptBase32ToString(remoteEncryptedKey, password);
|
|
} else if (
|
|
remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)
|
|
) {
|
|
key = await decryptBase64urlToString(remoteEncryptedKey, password);
|
|
} else {
|
|
throw Error(`unexpected key=${remoteEncryptedKey}`);
|
|
}
|
|
}
|
|
const backwardMapping = await getSyncMetaMappingByRemoteKey(
|
|
remoteType,
|
|
db,
|
|
key,
|
|
entry.lastModified,
|
|
entry.etag
|
|
);
|
|
|
|
let r = {} as FileOrFolderMixedState;
|
|
if (backwardMapping !== undefined) {
|
|
key = backwardMapping.localKey;
|
|
r = {
|
|
key: key,
|
|
exist_remote: true,
|
|
mtime_remote: backwardMapping.localMtime || entry.lastModified,
|
|
size_remote: backwardMapping.localSize || entry.size,
|
|
remote_encrypted_key: remoteEncryptedKey,
|
|
};
|
|
} else {
|
|
r = {
|
|
key: key,
|
|
exist_remote: true,
|
|
mtime_remote: entry.lastModified,
|
|
size_remote: entry.size,
|
|
remote_encrypted_key: remoteEncryptedKey,
|
|
};
|
|
}
|
|
if (isHiddenPath(key)) {
|
|
continue;
|
|
}
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].key = r.key;
|
|
results[key].exist_remote = r.exist_remote;
|
|
results[key].mtime_remote = r.mtime_remote;
|
|
results[key].size_remote = r.size_remote;
|
|
results[key].remote_encrypted_key = r.remote_encrypted_key;
|
|
} else {
|
|
results[key] = r;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const entry of local) {
|
|
let r = {} as FileOrFolderMixedState;
|
|
let key = entry.path;
|
|
|
|
if (entry.path === "/") {
|
|
// ignore
|
|
continue;
|
|
} else if (entry instanceof TFile) {
|
|
r = {
|
|
key: entry.path,
|
|
exist_local: true,
|
|
mtime_local: entry.stat.mtime,
|
|
size_local: entry.stat.size,
|
|
};
|
|
} else if (entry instanceof TFolder) {
|
|
key = `${entry.path}/`;
|
|
r = {
|
|
key: key,
|
|
exist_local: true,
|
|
mtime_local: undefined,
|
|
size_local: 0,
|
|
};
|
|
} else {
|
|
throw Error(`unexpected ${entry}`);
|
|
}
|
|
|
|
if (isHiddenPath(key)) {
|
|
continue;
|
|
}
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].key = r.key;
|
|
results[key].exist_local = r.exist_local;
|
|
results[key].mtime_local = r.mtime_local;
|
|
results[key].size_local = r.size_local;
|
|
} else {
|
|
results[key] = r;
|
|
}
|
|
}
|
|
|
|
for (const entry of deleteHistory) {
|
|
let key = entry.key;
|
|
if (entry.keyType === "folder") {
|
|
if (!entry.key.endsWith("/")) {
|
|
key = `${entry.key}/`;
|
|
}
|
|
} else if (entry.keyType === "file") {
|
|
// pass
|
|
} else {
|
|
throw Error(`unexpected ${entry}`);
|
|
}
|
|
|
|
const r = {
|
|
key: key,
|
|
delete_time_local: entry.actionWhen,
|
|
} as FileOrFolderMixedState;
|
|
|
|
if (isHiddenPath(key)) {
|
|
continue;
|
|
}
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].key = r.key;
|
|
results[key].delete_time_local = r.delete_time_local;
|
|
} else {
|
|
results[key] = r;
|
|
}
|
|
}
|
|
|
|
return results;
|
|
};
|
|
|
|
const getOperation = (
|
|
origRecord: FileOrFolderMixedState,
|
|
inplace: boolean = false,
|
|
password: string = ""
|
|
) => {
|
|
let r = origRecord;
|
|
if (!inplace) {
|
|
r = Object.assign({}, origRecord);
|
|
}
|
|
|
|
if (r.mtime_local === 0) {
|
|
r.mtime_local = undefined;
|
|
}
|
|
if (r.mtime_remote === 0) {
|
|
r.mtime_remote = undefined;
|
|
}
|
|
if (r.delete_time_local === 0) {
|
|
r.delete_time_local = undefined;
|
|
}
|
|
if (r.exist_local === undefined) {
|
|
r.exist_local = false;
|
|
}
|
|
if (r.exist_remote === undefined) {
|
|
r.exist_remote = false;
|
|
}
|
|
r.decision = "unknown";
|
|
|
|
if (
|
|
r.exist_remote &&
|
|
r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local !== undefined &&
|
|
r.mtime_remote > r.mtime_local
|
|
) {
|
|
r.decision = "download_clearhist";
|
|
r.decision_branch = 1;
|
|
} else if (
|
|
r.exist_remote &&
|
|
r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local !== undefined &&
|
|
r.mtime_remote < r.mtime_local
|
|
) {
|
|
r.decision = "upload_clearhist";
|
|
r.decision_branch = 2;
|
|
} else if (
|
|
r.exist_remote &&
|
|
r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local !== undefined &&
|
|
r.mtime_remote === r.mtime_local &&
|
|
password === "" &&
|
|
r.size_local === r.size_remote
|
|
) {
|
|
r.decision = "skip";
|
|
r.decision_branch = 3;
|
|
} else if (
|
|
r.exist_remote &&
|
|
r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local !== undefined &&
|
|
r.mtime_remote === r.mtime_local &&
|
|
password === "" &&
|
|
r.size_local !== r.size_remote
|
|
) {
|
|
r.decision = "upload_clearhist";
|
|
r.decision_branch = 4;
|
|
} else if (
|
|
r.exist_remote &&
|
|
r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local !== undefined &&
|
|
r.mtime_remote === r.mtime_local &&
|
|
password !== ""
|
|
) {
|
|
// if we have encryption,
|
|
// the size is always unequal
|
|
// only mtime(s) are reliable
|
|
r.decision = "skip";
|
|
r.decision_branch = 5;
|
|
} else if (r.exist_remote && r.exist_local && r.mtime_local === undefined) {
|
|
// this must be a folder!
|
|
if (!r.key.endsWith("/")) {
|
|
throw Error(`${r.key} is not a folder but lacks local mtime`);
|
|
}
|
|
r.decision = "skip";
|
|
r.decision_branch = 6;
|
|
} else if (
|
|
r.exist_remote &&
|
|
!r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local === undefined &&
|
|
r.delete_time_local !== undefined &&
|
|
r.mtime_remote >= r.delete_time_local
|
|
) {
|
|
r.decision = "download_clearhist";
|
|
r.decision_branch = 7;
|
|
} else if (
|
|
r.exist_remote &&
|
|
!r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local === undefined &&
|
|
r.delete_time_local !== undefined &&
|
|
r.mtime_remote < r.delete_time_local
|
|
) {
|
|
r.decision = "delremote_clearhist";
|
|
r.decision_branch = 8;
|
|
} else if (
|
|
r.exist_remote &&
|
|
!r.exist_local &&
|
|
r.mtime_remote !== undefined &&
|
|
r.mtime_local === undefined &&
|
|
r.delete_time_local == undefined
|
|
) {
|
|
r.decision = "download";
|
|
r.decision_branch = 9;
|
|
} else if (!r.exist_remote && r.exist_local && r.mtime_remote === undefined) {
|
|
r.decision = "upload_clearhist";
|
|
r.decision_branch = 10;
|
|
} else if (
|
|
!r.exist_remote &&
|
|
!r.exist_local &&
|
|
r.mtime_remote === undefined &&
|
|
r.mtime_local === undefined
|
|
) {
|
|
r.decision = "clearhist";
|
|
r.decision_branch = 11;
|
|
}
|
|
|
|
if (r.decision === "unknown") {
|
|
throw Error(`unknown decision for ${JSON.stringify(r)}`);
|
|
}
|
|
|
|
return r;
|
|
};
|
|
|
|
export const getSyncPlan = async (
|
|
remote: RemoteItem[],
|
|
local: TAbstractFile[],
|
|
deleteHistory: FileFolderHistoryRecord[],
|
|
db: InternalDBs,
|
|
remoteType: SUPPORTED_SERVICES_TYPE,
|
|
password: string = ""
|
|
) => {
|
|
const mixedStates = await ensembleMixedStates(
|
|
remote,
|
|
local,
|
|
deleteHistory,
|
|
db,
|
|
remoteType,
|
|
password
|
|
);
|
|
for (const [key, val] of Object.entries(mixedStates)) {
|
|
getOperation(val, true, password);
|
|
}
|
|
const plan = {
|
|
ts: Date.now(),
|
|
remoteType: remoteType,
|
|
mixedStates: mixedStates,
|
|
} as SyncPlanType;
|
|
return plan;
|
|
};
|
|
|
|
const dispatchOperationToActual = async (
|
|
key: string,
|
|
state: FileOrFolderMixedState,
|
|
client: RemoteClient,
|
|
db: InternalDBs,
|
|
vault: Vault,
|
|
password: string = "",
|
|
foldersCreatedBefore: Set<string> | undefined = undefined
|
|
) => {
|
|
let remoteEncryptedKey = key;
|
|
if (password !== "") {
|
|
remoteEncryptedKey = state.remote_encrypted_key;
|
|
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
|
// the old version uses base32
|
|
// remoteEncryptedKey = await encryptStringToBase32(key, password);
|
|
// the new version users base64url
|
|
remoteEncryptedKey = await encryptStringToBase64url(key, password);
|
|
}
|
|
}
|
|
|
|
if (
|
|
state.decision === undefined ||
|
|
state.decision === "unknown" ||
|
|
state.decision === "undecided"
|
|
) {
|
|
throw Error(`unknown decision in ${JSON.stringify(state)}`);
|
|
} else if (state.decision === "skip") {
|
|
// do nothing
|
|
} else if (state.decision === "download_clearhist") {
|
|
await client.downloadFromRemote(
|
|
state.key,
|
|
vault,
|
|
state.mtime_remote,
|
|
password,
|
|
remoteEncryptedKey
|
|
);
|
|
await clearDeleteRenameHistoryOfKey(db, state.key);
|
|
} else if (state.decision === "upload_clearhist") {
|
|
const remoteObjMeta = await client.uploadToRemote(
|
|
state.key,
|
|
vault,
|
|
false,
|
|
password,
|
|
remoteEncryptedKey,
|
|
foldersCreatedBefore
|
|
);
|
|
await upsertSyncMetaMappingData(
|
|
client.serviceType,
|
|
db,
|
|
state.key,
|
|
state.mtime_local,
|
|
state.size_local,
|
|
state.key,
|
|
remoteObjMeta.lastModified,
|
|
remoteObjMeta.size,
|
|
remoteObjMeta.etag
|
|
);
|
|
await clearDeleteRenameHistoryOfKey(db, state.key);
|
|
} else if (state.decision === "download") {
|
|
await mkdirpInVault(state.key, vault);
|
|
await client.downloadFromRemote(
|
|
state.key,
|
|
vault,
|
|
state.mtime_remote,
|
|
password,
|
|
remoteEncryptedKey
|
|
);
|
|
} else if (state.decision === "delremote_clearhist") {
|
|
await client.deleteFromRemote(state.key, password, remoteEncryptedKey);
|
|
await clearDeleteRenameHistoryOfKey(db, state.key);
|
|
} else if (state.decision === "upload") {
|
|
const remoteObjMeta = await client.uploadToRemote(
|
|
state.key,
|
|
vault,
|
|
false,
|
|
password,
|
|
remoteEncryptedKey,
|
|
foldersCreatedBefore
|
|
);
|
|
await upsertSyncMetaMappingData(
|
|
client.serviceType,
|
|
db,
|
|
state.key,
|
|
state.mtime_local,
|
|
state.size_local,
|
|
state.key,
|
|
remoteObjMeta.lastModified,
|
|
remoteObjMeta.size,
|
|
remoteObjMeta.etag
|
|
);
|
|
} else if (state.decision === "clearhist") {
|
|
await clearDeleteRenameHistoryOfKey(db, state.key);
|
|
} else {
|
|
throw Error("this should never happen!");
|
|
}
|
|
};
|
|
|
|
export const doActualSync = async (
|
|
client: RemoteClient,
|
|
db: InternalDBs,
|
|
vault: Vault,
|
|
syncPlan: SyncPlanType,
|
|
password: string = ""
|
|
) => {
|
|
const keyStates = syncPlan.mixedStates;
|
|
const foldersCreatedBefore = new Set<string>();
|
|
for (const [k, v] of Object.entries(keyStates).sort(
|
|
([k1, v1], [k2, v2]) => k2.length - k1.length
|
|
)) {
|
|
const k2 = k as string;
|
|
const v2 = v as FileOrFolderMixedState;
|
|
await dispatchOperationToActual(
|
|
k as string,
|
|
v as FileOrFolderMixedState,
|
|
client,
|
|
db,
|
|
vault,
|
|
password,
|
|
foldersCreatedBefore
|
|
);
|
|
// log.info(`finished ${k}, with ${setToString(foldersCreatedBefore)}`);
|
|
}
|
|
// await Promise.all(
|
|
// Object.entries(keyStates)
|
|
// .map(async ([k, v]) =>
|
|
// dispatchOperationToActual(
|
|
// k as string,
|
|
// v as FileOrFolderMixedState,
|
|
// client,
|
|
// db,
|
|
// vault,
|
|
// password,
|
|
// foldersCreatedBefore
|
|
// )
|
|
// )
|
|
// );
|
|
};
|