1226 lines
35 KiB
TypeScript
1226 lines
35 KiB
TypeScript
import {
|
|
TAbstractFile,
|
|
TFile,
|
|
TFolder,
|
|
Vault,
|
|
requireApiVersion,
|
|
} from "obsidian";
|
|
import AggregateError from "aggregate-error";
|
|
import PQueue from "p-queue";
|
|
import {
|
|
RemoteItem,
|
|
SUPPORTED_SERVICES_TYPE,
|
|
DecisionType,
|
|
FileOrFolderMixedState,
|
|
API_VER_STAT_FOLDER,
|
|
} from "./baseTypes";
|
|
import {
|
|
decryptBase32ToString,
|
|
decryptBase64urlToString,
|
|
encryptStringToBase64url,
|
|
MAGIC_ENCRYPTED_PREFIX_BASE32,
|
|
MAGIC_ENCRYPTED_PREFIX_BASE64URL,
|
|
} from "./encrypt";
|
|
import type { FileFolderHistoryRecord, InternalDBs } from "./localdb";
|
|
import {
|
|
clearDeleteRenameHistoryOfKeyAndVault,
|
|
getSyncMetaMappingByRemoteKeyAndVault,
|
|
upsertSyncMetaMappingDataByVault,
|
|
} from "./localdb";
|
|
import {
|
|
isHiddenPath,
|
|
isVaildText,
|
|
mkdirpInVault,
|
|
getFolderLevels,
|
|
getParentFolder,
|
|
atWhichLevel,
|
|
unixTimeToStr,
|
|
} from "./misc";
|
|
import { RemoteClient } from "./remote";
|
|
import {
|
|
MetadataOnRemote,
|
|
DeletionOnRemote,
|
|
serializeMetadataOnRemote,
|
|
deserializeMetadataOnRemote,
|
|
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE,
|
|
DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2,
|
|
isEqualMetadataOnRemote,
|
|
} from "./metadataOnRemote";
|
|
import { isInsideObsFolder, ObsConfigDirFileType } from "./obsFolderLister";
|
|
|
|
import { log } from "./moreOnLog";
|
|
|
|
export type SyncStatusType =
|
|
| "idle"
|
|
| "preparing"
|
|
| "getting_remote_files_list"
|
|
| "getting_remote_extra_meta"
|
|
| "getting_local_meta"
|
|
| "checking_password"
|
|
| "generating_plan"
|
|
| "syncing"
|
|
| "cleaning"
|
|
| "finish";
|
|
|
|
export interface SyncPlanType {
|
|
ts: number;
|
|
tsFmt?: string;
|
|
remoteType: SUPPORTED_SERVICES_TYPE;
|
|
mixedStates: Record<string, FileOrFolderMixedState>;
|
|
}
|
|
|
|
export interface PasswordCheckType {
|
|
ok: boolean;
|
|
reason:
|
|
| "ok"
|
|
| "empty_remote"
|
|
| "remote_encrypted_local_no_password"
|
|
| "password_matched"
|
|
| "password_not_matched"
|
|
| "invalid_text_after_decryption"
|
|
| "remote_not_encrypted_local_has_password"
|
|
| "no_password_both_sides";
|
|
}
|
|
|
|
export const isPasswordOk = async (
|
|
remote: RemoteItem[],
|
|
password: string = ""
|
|
) => {
|
|
if (remote === undefined || remote.length === 0) {
|
|
// remote empty
|
|
return {
|
|
ok: true,
|
|
reason: "empty_remote",
|
|
} as PasswordCheckType;
|
|
}
|
|
const santyCheckKey = remote[0].key;
|
|
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
|
// this is encrypted using old base32!
|
|
// try to decrypt it using the provided password.
|
|
if (password === "") {
|
|
return {
|
|
ok: false,
|
|
reason: "remote_encrypted_local_no_password",
|
|
} as PasswordCheckType;
|
|
}
|
|
try {
|
|
const res = await decryptBase32ToString(santyCheckKey, password);
|
|
|
|
// additional test
|
|
// because iOS Safari bypasses decryption with wrong password!
|
|
if (isVaildText(res)) {
|
|
return {
|
|
ok: true,
|
|
reason: "password_matched",
|
|
} as PasswordCheckType;
|
|
} else {
|
|
return {
|
|
ok: false,
|
|
reason: "invalid_text_after_decryption",
|
|
} as PasswordCheckType;
|
|
}
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
reason: "password_not_matched",
|
|
} as PasswordCheckType;
|
|
}
|
|
}
|
|
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
|
|
// this is encrypted using new base64url!
|
|
// try to decrypt it using the provided password.
|
|
if (password === "") {
|
|
return {
|
|
ok: false,
|
|
reason: "remote_encrypted_local_no_password",
|
|
} as PasswordCheckType;
|
|
}
|
|
try {
|
|
const res = await decryptBase64urlToString(santyCheckKey, password);
|
|
|
|
// additional test
|
|
// because iOS Safari bypasses decryption with wrong password!
|
|
if (isVaildText(res)) {
|
|
return {
|
|
ok: true,
|
|
reason: "password_matched",
|
|
} as PasswordCheckType;
|
|
} else {
|
|
return {
|
|
ok: false,
|
|
reason: "invalid_text_after_decryption",
|
|
} as PasswordCheckType;
|
|
}
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
reason: "password_not_matched",
|
|
} as PasswordCheckType;
|
|
}
|
|
} else {
|
|
// it is not encrypted!
|
|
if (password !== "") {
|
|
return {
|
|
ok: false,
|
|
reason: "remote_not_encrypted_local_has_password",
|
|
} as PasswordCheckType;
|
|
}
|
|
return {
|
|
ok: true,
|
|
reason: "no_password_both_sides",
|
|
} as PasswordCheckType;
|
|
}
|
|
};
|
|
|
|
export const parseRemoteItems = async (
|
|
remote: RemoteItem[],
|
|
db: InternalDBs,
|
|
vaultRandomID: string,
|
|
remoteType: SUPPORTED_SERVICES_TYPE,
|
|
password: string = ""
|
|
) => {
|
|
const remoteStates = [] as FileOrFolderMixedState[];
|
|
let metadataFile: FileOrFolderMixedState = undefined;
|
|
if (remote === undefined) {
|
|
return {
|
|
remoteStates: remoteStates,
|
|
metadataFile: metadataFile,
|
|
};
|
|
}
|
|
|
|
for (const entry of remote) {
|
|
const remoteEncryptedKey = entry.key;
|
|
let key = remoteEncryptedKey;
|
|
if (password !== "") {
|
|
if (remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
|
|
key = await decryptBase32ToString(remoteEncryptedKey, password);
|
|
} else if (
|
|
remoteEncryptedKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)
|
|
) {
|
|
key = await decryptBase64urlToString(remoteEncryptedKey, password);
|
|
} else {
|
|
throw Error(`unexpected key=${remoteEncryptedKey}`);
|
|
}
|
|
}
|
|
const backwardMapping = await getSyncMetaMappingByRemoteKeyAndVault(
|
|
remoteType,
|
|
db,
|
|
key,
|
|
entry.lastModified,
|
|
entry.etag,
|
|
vaultRandomID
|
|
);
|
|
|
|
let r = {} as FileOrFolderMixedState;
|
|
if (backwardMapping !== undefined) {
|
|
key = backwardMapping.localKey;
|
|
const mtimeRemote = backwardMapping.localMtime || entry.lastModified;
|
|
r = {
|
|
key: key,
|
|
existRemote: true,
|
|
mtimeRemote: mtimeRemote,
|
|
mtimeRemoteFmt: unixTimeToStr(mtimeRemote),
|
|
sizeRemote: backwardMapping.localSize || entry.size,
|
|
remoteEncryptedKey: remoteEncryptedKey,
|
|
changeRemoteMtimeUsingMapping: true,
|
|
};
|
|
} else {
|
|
r = {
|
|
key: key,
|
|
existRemote: true,
|
|
mtimeRemote: entry.lastModified,
|
|
mtimeRemoteFmt: unixTimeToStr(entry.lastModified),
|
|
sizeRemote: entry.size,
|
|
remoteEncryptedKey: remoteEncryptedKey,
|
|
changeRemoteMtimeUsingMapping: false,
|
|
};
|
|
}
|
|
|
|
if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE) {
|
|
metadataFile = Object.assign({}, r);
|
|
}
|
|
if (r.key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2) {
|
|
throw Error(
|
|
`A reserved file name ${r.key} has been found. You may upgrade the plugin to latest version to try to deal with it.`
|
|
);
|
|
}
|
|
|
|
remoteStates.push(r);
|
|
}
|
|
return {
|
|
remoteStates: remoteStates,
|
|
metadataFile: metadataFile,
|
|
};
|
|
};
|
|
|
|
export const fetchMetadataFile = async (
|
|
metadataFile: FileOrFolderMixedState,
|
|
client: RemoteClient,
|
|
vault: Vault,
|
|
password: string = ""
|
|
) => {
|
|
if (metadataFile === undefined) {
|
|
log.debug("no metadata file, so no fetch");
|
|
return {
|
|
deletions: [],
|
|
} as MetadataOnRemote;
|
|
}
|
|
|
|
const buf = await client.downloadFromRemote(
|
|
metadataFile.key,
|
|
vault,
|
|
metadataFile.mtimeRemote,
|
|
password,
|
|
metadataFile.remoteEncryptedKey,
|
|
true
|
|
);
|
|
const metadata = deserializeMetadataOnRemote(buf);
|
|
return metadata;
|
|
};
|
|
|
|
const isSkipItem = (
|
|
key: string,
|
|
syncConfigDir: boolean,
|
|
syncUnderscoreItems: boolean,
|
|
configDir: string
|
|
) => {
|
|
if (syncConfigDir && isInsideObsFolder(key, configDir)) {
|
|
return false;
|
|
}
|
|
return (
|
|
isHiddenPath(key, true, false) ||
|
|
(!syncUnderscoreItems && isHiddenPath(key, false, true)) ||
|
|
key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE ||
|
|
key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2
|
|
);
|
|
};
|
|
|
|
const ensembleMixedStates = async (
|
|
remoteStates: FileOrFolderMixedState[],
|
|
local: TAbstractFile[],
|
|
localConfigDirContents: ObsConfigDirFileType[] | undefined,
|
|
remoteDeleteHistory: DeletionOnRemote[],
|
|
localFileHistory: FileFolderHistoryRecord[],
|
|
syncConfigDir: boolean,
|
|
configDir: string,
|
|
syncUnderscoreItems: boolean
|
|
) => {
|
|
const results = {} as Record<string, FileOrFolderMixedState>;
|
|
|
|
for (const r of remoteStates) {
|
|
const key = r.key;
|
|
|
|
if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
|
|
continue;
|
|
}
|
|
results[key] = r;
|
|
results[key].existLocal = false;
|
|
}
|
|
|
|
for (const entry of local) {
|
|
let r = {} as FileOrFolderMixedState;
|
|
let key = entry.path;
|
|
|
|
if (entry.path === "/") {
|
|
// ignore
|
|
continue;
|
|
} else if (entry instanceof TFile) {
|
|
const mtimeLocal = Math.max(entry.stat.mtime || 0, entry.stat.ctime || 0);
|
|
log.info(`[pos1], ${entry.path}, mtimeLocal=${mtimeLocal}, mtime=${entry.stat.mtime}, ctime=${entry.stat.ctime}`);
|
|
r = {
|
|
key: entry.path,
|
|
existLocal: true,
|
|
mtimeLocal: mtimeLocal,
|
|
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
|
|
sizeLocal: entry.stat.size,
|
|
};
|
|
} else if (entry instanceof TFolder) {
|
|
key = `${entry.path}/`;
|
|
r = {
|
|
key: key,
|
|
existLocal: true,
|
|
mtimeLocal: undefined,
|
|
mtimeLocalFmt: undefined,
|
|
sizeLocal: 0,
|
|
};
|
|
} else {
|
|
throw Error(`unexpected ${entry}`);
|
|
}
|
|
|
|
if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
|
|
continue;
|
|
}
|
|
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].key = r.key;
|
|
results[key].existLocal = r.existLocal;
|
|
results[key].mtimeLocal = r.mtimeLocal;
|
|
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
|
|
results[key].sizeLocal = r.sizeLocal;
|
|
} else {
|
|
results[key] = r;
|
|
results[key].existRemote = false;
|
|
}
|
|
log.info(`[pos2], ${key}, mtimeLocal=${results[key].mtimeLocal}`);
|
|
}
|
|
|
|
if (syncConfigDir && localConfigDirContents !== undefined) {
|
|
for (const entry of localConfigDirContents) {
|
|
const key = entry.key;
|
|
const mtimeLocal = Math.max(entry.mtime, entry.ctime);
|
|
const r: FileOrFolderMixedState = {
|
|
key: key,
|
|
existLocal: true,
|
|
mtimeLocal: mtimeLocal,
|
|
mtimeLocalFmt: unixTimeToStr(mtimeLocal),
|
|
sizeLocal: entry.size,
|
|
};
|
|
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].key = r.key;
|
|
results[key].existLocal = r.existLocal;
|
|
results[key].mtimeLocal = r.mtimeLocal;
|
|
results[key].mtimeLocalFmt = r.mtimeLocalFmt;
|
|
results[key].sizeLocal = r.sizeLocal;
|
|
} else {
|
|
results[key] = r;
|
|
results[key].existRemote = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const entry of remoteDeleteHistory) {
|
|
const key = entry.key;
|
|
const r = {
|
|
key: key,
|
|
deltimeRemote: entry.actionWhen,
|
|
deltimeRemoteFmt: unixTimeToStr(entry.actionWhen),
|
|
} as FileOrFolderMixedState;
|
|
|
|
if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
|
|
continue;
|
|
}
|
|
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].key = r.key;
|
|
results[key].deltimeRemote = r.deltimeRemote;
|
|
results[key].deltimeRemoteFmt = r.deltimeRemoteFmt;
|
|
} else {
|
|
results[key] = r;
|
|
|
|
results[key].existLocal = false;
|
|
results[key].existRemote = false;
|
|
}
|
|
log.info(`[pos3], ${key}, mtimeLocal=${results[key].mtimeLocal}`);
|
|
}
|
|
|
|
for (const entry of localFileHistory) {
|
|
let key = entry.key;
|
|
if (entry.keyType === "folder") {
|
|
if (!entry.key.endsWith("/")) {
|
|
key = `${entry.key}/`;
|
|
}
|
|
} else if (entry.keyType === "file") {
|
|
// pass
|
|
} else {
|
|
throw Error(`unexpected ${entry}`);
|
|
}
|
|
|
|
if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
|
|
continue;
|
|
}
|
|
|
|
if (entry.actionType === "delete" || entry.actionType === "rename") {
|
|
const r = {
|
|
key: key,
|
|
deltimeLocal: entry.actionWhen,
|
|
deltimeLocalFmt: unixTimeToStr(entry.actionWhen),
|
|
} as FileOrFolderMixedState;
|
|
|
|
if (results.hasOwnProperty(key)) {
|
|
results[key].deltimeLocal = r.deltimeLocal;
|
|
results[key].deltimeLocalFmt = r.deltimeLocalFmt;
|
|
} else {
|
|
results[key] = r;
|
|
results[key].existLocal = false; // we have already checked local
|
|
results[key].existRemote = false; // we have already checked remote
|
|
}
|
|
} else if (entry.actionType === "renameDestination") {
|
|
const r = {
|
|
key: key,
|
|
mtimeLocal: entry.actionWhen,
|
|
mtimeLocalFmt: unixTimeToStr(entry.actionWhen),
|
|
changeLocalMtimeUsingMapping: true,
|
|
};
|
|
log.info(`[pos4], ${key}, mtimeLocal=${results[key].mtimeLocal}, changeLocalMtimeUsingMapping=true`);
|
|
if (results.hasOwnProperty(key)) {
|
|
const mtimeLocal = Math.max(
|
|
r.mtimeLocal || 0,
|
|
results[key].mtimeLocal || 0
|
|
);
|
|
results[key].mtimeLocal = mtimeLocal;
|
|
results[key].mtimeLocalFmt = unixTimeToStr(mtimeLocal);
|
|
results[key].changeLocalMtimeUsingMapping =
|
|
r.changeLocalMtimeUsingMapping;
|
|
} else {
|
|
results[key] = r;
|
|
results[key].existLocal = false; // we have already checked local
|
|
results[key].existRemote = false; // we have already checked remote
|
|
}
|
|
log.info(`[pos5], ${key}, mtimeLocal=${results[key].mtimeLocal},changeLocalMtimeUsingMapping=${results[key].changeLocalMtimeUsingMapping} `);
|
|
} else {
|
|
throw Error(
|
|
`do not know how to deal with local file history ${entry.key} with ${entry.actionType}`
|
|
);
|
|
}
|
|
}
|
|
|
|
log.info(`[pos6]`);
|
|
log.info(results);
|
|
return results;
|
|
};
|
|
|
|
const assignOperationToFileInplace = (
|
|
origRecord: FileOrFolderMixedState,
|
|
keptFolder: Set<string>,
|
|
password: string = ""
|
|
) => {
|
|
let r = origRecord;
|
|
|
|
// files and folders are treated differently
|
|
// here we only check files
|
|
if (r.key.endsWith("/")) {
|
|
return r;
|
|
}
|
|
|
|
// we find the max date from four sources
|
|
|
|
// 0. find anything inconsistent
|
|
if (r.existLocal && (r.mtimeLocal === undefined || r.mtimeLocal <= 0)) {
|
|
throw Error(
|
|
`Error: Abnormal last modified time locally: ${JSON.stringify(
|
|
r,
|
|
null,
|
|
2
|
|
)}`
|
|
);
|
|
}
|
|
if (r.existRemote && (r.mtimeRemote === undefined || r.mtimeRemote <= 0)) {
|
|
throw Error(
|
|
`Error: Abnormal last modified time remotely: ${JSON.stringify(
|
|
r,
|
|
null,
|
|
2
|
|
)}`
|
|
);
|
|
}
|
|
if (r.deltimeLocal !== undefined && r.deltimeLocal <= 0) {
|
|
throw Error(
|
|
`Error: Abnormal deletion time locally: ${JSON.stringify(r, null, 2)}`
|
|
);
|
|
}
|
|
if (r.deltimeRemote !== undefined && r.deltimeRemote <= 0) {
|
|
throw Error(
|
|
`Error: Abnormal deletion time remotely: ${JSON.stringify(r, null, 2)}`
|
|
);
|
|
}
|
|
|
|
// 1. mtimeLocal
|
|
if (r.existLocal) {
|
|
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
|
|
const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
|
|
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
|
|
if (
|
|
r.mtimeLocal >= mtimeRemote &&
|
|
r.mtimeLocal >= deltimeLocal &&
|
|
r.mtimeLocal >= deltimeRemote
|
|
) {
|
|
if (r.mtimeLocal === r.mtimeRemote) {
|
|
// mtime the same
|
|
if (password === "") {
|
|
// no password, we can also compare the sizes!
|
|
if (r.sizeLocal === r.sizeRemote) {
|
|
r.decision = "skipUploading";
|
|
r.decisionBranch = 1;
|
|
} else {
|
|
r.decision = "uploadLocalToRemote";
|
|
r.decisionBranch = 2;
|
|
}
|
|
} else {
|
|
// we have password, then the sizes are always unequal
|
|
// we can only rely on mtime
|
|
r.decision = "skipUploading";
|
|
r.decisionBranch = 3;
|
|
}
|
|
} else {
|
|
r.decision = "uploadLocalToRemote";
|
|
r.decisionBranch = 4;
|
|
}
|
|
keptFolder.add(getParentFolder(r.key));
|
|
return r;
|
|
}
|
|
}
|
|
|
|
// 2. mtimeRemote
|
|
if (r.existRemote) {
|
|
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
|
|
const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
|
|
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
|
|
if (
|
|
r.mtimeRemote > mtimeLocal &&
|
|
r.mtimeRemote >= deltimeLocal &&
|
|
r.mtimeRemote >= deltimeRemote
|
|
) {
|
|
r.decision = "downloadRemoteToLocal";
|
|
r.decisionBranch = 5;
|
|
keptFolder.add(getParentFolder(r.key));
|
|
return r;
|
|
}
|
|
}
|
|
|
|
// 3. deltimeLocal
|
|
if (r.deltimeLocal !== undefined && r.deltimeLocal !== 0) {
|
|
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
|
|
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
|
|
const deltimeRemote = r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
|
|
if (
|
|
r.deltimeLocal >= mtimeLocal &&
|
|
r.deltimeLocal >= mtimeRemote &&
|
|
r.deltimeLocal >= deltimeRemote
|
|
) {
|
|
r.decision = "uploadLocalDelHistToRemote";
|
|
r.decisionBranch = 6;
|
|
if (r.existLocal || r.existRemote) {
|
|
// actual deletion would happen
|
|
}
|
|
return r;
|
|
}
|
|
}
|
|
|
|
// 4. deltimeRemote
|
|
if (r.deltimeRemote !== undefined && r.deltimeRemote !== 0) {
|
|
const mtimeLocal = r.existLocal ? r.mtimeLocal : -1;
|
|
const mtimeRemote = r.existRemote ? r.mtimeRemote : -1;
|
|
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
|
|
if (
|
|
r.deltimeRemote >= mtimeLocal &&
|
|
r.deltimeRemote >= mtimeRemote &&
|
|
r.deltimeRemote >= deltimeLocal
|
|
) {
|
|
r.decision = "keepRemoteDelHist";
|
|
r.decisionBranch = 7;
|
|
if (r.existLocal || r.existRemote) {
|
|
// actual deletion would happen
|
|
}
|
|
return r;
|
|
}
|
|
}
|
|
|
|
throw Error(`no decision for ${JSON.stringify(r)}`);
|
|
};
|
|
|
|
const assignOperationToFolderInplace = async (
|
|
origRecord: FileOrFolderMixedState,
|
|
keptFolder: Set<string>,
|
|
vault: Vault,
|
|
password: string = ""
|
|
) => {
|
|
let r = origRecord;
|
|
|
|
// files and folders are treated differently
|
|
// here we only check folders
|
|
if (!r.key.endsWith("/")) {
|
|
return r;
|
|
}
|
|
|
|
if (!keptFolder.has(r.key)) {
|
|
// the folder does NOT have any must-be-kept children!
|
|
|
|
if (r.deltimeLocal !== undefined || r.deltimeRemote !== undefined) {
|
|
// it has some deletion "commands"
|
|
|
|
const deltimeLocal = r.deltimeLocal !== undefined ? r.deltimeLocal : -1;
|
|
const deltimeRemote =
|
|
r.deltimeRemote !== undefined ? r.deltimeRemote : -1;
|
|
|
|
// if it was created after deletion, we should keep it as is
|
|
if (requireApiVersion(API_VER_STAT_FOLDER)) {
|
|
if (r.existLocal) {
|
|
const { ctime, mtime } = await vault.adapter.stat(r.key);
|
|
const cmtime = Math.max(ctime, mtime);
|
|
if (cmtime > 0 && cmtime >= deltimeLocal && cmtime >= deltimeRemote) {
|
|
keptFolder.add(getParentFolder(r.key));
|
|
if (r.existLocal && r.existRemote) {
|
|
r.decision = "skipFolder";
|
|
r.decisionBranch = 14;
|
|
} else if (r.existLocal || r.existRemote) {
|
|
r.decision = "createFolder";
|
|
r.decisionBranch = 15;
|
|
} else {
|
|
throw Error(
|
|
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// If it was moved to here, after deletion, we should keep it as is.
|
|
// The logic not necessarily needs API_VER_STAT_FOLDER.
|
|
// The folder needs this logic because it's also determined by file children.
|
|
// But the file do not need this logic because the mtimeLocal is checked firstly.
|
|
if (
|
|
r.existLocal &&
|
|
r.changeLocalMtimeUsingMapping &&
|
|
r.mtimeLocal > 0 &&
|
|
r.mtimeLocal > deltimeLocal &&
|
|
r.mtimeLocal > deltimeRemote
|
|
) {
|
|
keptFolder.add(getParentFolder(r.key));
|
|
if (r.existLocal && r.existRemote) {
|
|
r.decision = "skipFolder";
|
|
r.decisionBranch = 16;
|
|
} else if (r.existLocal || r.existRemote) {
|
|
r.decision = "createFolder";
|
|
r.decisionBranch = 17;
|
|
} else {
|
|
throw Error(
|
|
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
|
|
);
|
|
}
|
|
}
|
|
|
|
if (r.decision === undefined) {
|
|
// not yet decided by the above reason
|
|
if (deltimeLocal > 0 && deltimeLocal > deltimeRemote) {
|
|
r.decision = "uploadLocalDelHistToRemoteFolder";
|
|
r.decisionBranch = 8;
|
|
} else {
|
|
r.decision = "keepRemoteDelHistFolder";
|
|
r.decisionBranch = 9;
|
|
}
|
|
}
|
|
} else {
|
|
// it does not have any deletion commands
|
|
// keep it as is, and create it if necessary
|
|
keptFolder.add(getParentFolder(r.key));
|
|
if (r.existLocal && r.existRemote) {
|
|
r.decision = "skipFolder";
|
|
r.decisionBranch = 10;
|
|
} else if (r.existLocal || r.existRemote) {
|
|
r.decision = "createFolder";
|
|
r.decisionBranch = 11;
|
|
} else {
|
|
throw Error(
|
|
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
|
|
);
|
|
}
|
|
}
|
|
} else {
|
|
// the folder has some must be kept children!
|
|
// so itself and its parent folder must be kept
|
|
keptFolder.add(getParentFolder(r.key));
|
|
if (r.existLocal && r.existRemote) {
|
|
r.decision = "skipFolder";
|
|
r.decisionBranch = 12;
|
|
} else if (r.existLocal || r.existRemote) {
|
|
r.decision = "createFolder";
|
|
r.decisionBranch = 13;
|
|
} else {
|
|
throw Error(
|
|
`Error: Folder ${r.key} doesn't exist locally and remotely but is marked must be kept. Abort.`
|
|
);
|
|
}
|
|
}
|
|
|
|
// save the memory, save the world!
|
|
// we have dealt with it, so we don't need it any more.
|
|
keptFolder.delete(r.key);
|
|
return r;
|
|
};
|
|
|
|
const DELETION_DECISIONS: Set<DecisionType> = new Set([
|
|
"uploadLocalDelHistToRemote",
|
|
"keepRemoteDelHist",
|
|
"uploadLocalDelHistToRemoteFolder",
|
|
"keepRemoteDelHistFolder",
|
|
]);
|
|
|
|
export const getSyncPlan = async (
|
|
remoteStates: FileOrFolderMixedState[],
|
|
local: TAbstractFile[],
|
|
localConfigDirContents: ObsConfigDirFileType[] | undefined,
|
|
remoteDeleteHistory: DeletionOnRemote[],
|
|
localFileHistory: FileFolderHistoryRecord[],
|
|
remoteType: SUPPORTED_SERVICES_TYPE,
|
|
vault: Vault,
|
|
syncConfigDir: boolean,
|
|
configDir: string,
|
|
syncUnderscoreItems: boolean,
|
|
password: string = ""
|
|
) => {
|
|
const mixedStates = await ensembleMixedStates(
|
|
remoteStates,
|
|
local,
|
|
localConfigDirContents,
|
|
remoteDeleteHistory,
|
|
localFileHistory,
|
|
syncConfigDir,
|
|
configDir,
|
|
syncUnderscoreItems
|
|
);
|
|
|
|
const sortedKeys = Object.keys(mixedStates).sort(
|
|
(k1, k2) => k2.length - k1.length
|
|
);
|
|
|
|
const deletions: DeletionOnRemote[] = [];
|
|
|
|
const keptFolder = new Set<string>();
|
|
for (let i = 0; i < sortedKeys.length; ++i) {
|
|
const key = sortedKeys[i];
|
|
const val = mixedStates[key];
|
|
|
|
if (key.endsWith("/")) {
|
|
// decide some folders
|
|
// because the keys are sorted by length
|
|
// so all the children must have been shown up before in the iteration
|
|
await assignOperationToFolderInplace(val, keptFolder, vault, password);
|
|
} else {
|
|
// get all operations of files
|
|
// and at the same time get some helper info for folders
|
|
assignOperationToFileInplace(val, keptFolder, password);
|
|
}
|
|
|
|
if (DELETION_DECISIONS.has(val.decision)) {
|
|
if (val.decision === "uploadLocalDelHistToRemote") {
|
|
deletions.push({
|
|
key: key,
|
|
actionWhen: val.deltimeLocal,
|
|
});
|
|
} else if (val.decision === "keepRemoteDelHist") {
|
|
deletions.push({
|
|
key: key,
|
|
actionWhen: val.deltimeRemote,
|
|
});
|
|
} else if (val.decision === "uploadLocalDelHistToRemoteFolder") {
|
|
deletions.push({
|
|
key: key,
|
|
actionWhen: val.deltimeLocal,
|
|
});
|
|
} else if (val.decision === "keepRemoteDelHistFolder") {
|
|
deletions.push({
|
|
key: key,
|
|
actionWhen: val.deltimeRemote,
|
|
});
|
|
} else {
|
|
throw Error(`do not know how to delete for decision ${val.decision}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const currTs = Date.now();
|
|
const currTsFmt = unixTimeToStr(currTs);
|
|
const plan = {
|
|
ts: currTs,
|
|
tsFmt: currTsFmt,
|
|
remoteType: remoteType,
|
|
mixedStates: mixedStates,
|
|
} as SyncPlanType;
|
|
return {
|
|
plan: plan,
|
|
sortedKeys: sortedKeys,
|
|
deletions: deletions,
|
|
};
|
|
};
|
|
|
|
const uploadExtraMeta = async (
|
|
client: RemoteClient,
|
|
metadataFile: FileOrFolderMixedState | undefined,
|
|
origMetadata: MetadataOnRemote | undefined,
|
|
deletions: DeletionOnRemote[],
|
|
password: string = ""
|
|
) => {
|
|
if (deletions === undefined || deletions.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const key = DEFAULT_FILE_NAME_FOR_METADATAONREMOTE;
|
|
let remoteEncryptedKey = key;
|
|
|
|
if (password !== "") {
|
|
if (metadataFile === undefined) {
|
|
remoteEncryptedKey = undefined;
|
|
} else {
|
|
remoteEncryptedKey = metadataFile.remoteEncryptedKey;
|
|
}
|
|
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
|
// remoteEncryptedKey = await encryptStringToBase32(key, password);
|
|
remoteEncryptedKey = await encryptStringToBase64url(key, password);
|
|
}
|
|
}
|
|
|
|
const newMetadata: MetadataOnRemote = {
|
|
deletions: deletions,
|
|
};
|
|
|
|
if (isEqualMetadataOnRemote(origMetadata, newMetadata)) {
|
|
log.debug(
|
|
"metadata are the same, no need to re-generate and re-upload it."
|
|
);
|
|
return;
|
|
}
|
|
|
|
const resultText = serializeMetadataOnRemote(newMetadata);
|
|
|
|
await client.uploadToRemote(
|
|
key,
|
|
undefined,
|
|
false,
|
|
password,
|
|
remoteEncryptedKey,
|
|
undefined,
|
|
true,
|
|
resultText
|
|
);
|
|
};
|
|
|
|
const dispatchOperationToActual = async (
|
|
key: string,
|
|
vaultRandomID: string,
|
|
r: FileOrFolderMixedState,
|
|
client: RemoteClient,
|
|
db: InternalDBs,
|
|
vault: Vault,
|
|
localDeleteFunc: any,
|
|
password: string = ""
|
|
) => {
|
|
let remoteEncryptedKey = key;
|
|
if (password !== "") {
|
|
remoteEncryptedKey = r.remoteEncryptedKey;
|
|
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
|
|
// the old version uses base32
|
|
// remoteEncryptedKey = await encryptStringToBase32(key, password);
|
|
// the new version users base64url
|
|
remoteEncryptedKey = await encryptStringToBase64url(key, password);
|
|
}
|
|
}
|
|
|
|
if (r.decision === undefined) {
|
|
throw Error(`unknown decision in ${JSON.stringify(r)}`);
|
|
} else if (r.decision === "skipUploading") {
|
|
// do nothing!
|
|
} else if (r.decision === "uploadLocalDelHistToRemote") {
|
|
if (r.existLocal) {
|
|
await localDeleteFunc(r.key);
|
|
}
|
|
if (r.existRemote) {
|
|
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
|
|
}
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "keepRemoteDelHist") {
|
|
if (r.existLocal) {
|
|
await localDeleteFunc(r.key);
|
|
}
|
|
if (r.existRemote) {
|
|
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
|
|
}
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "uploadLocalToRemote") {
|
|
if (
|
|
client.serviceType === "onedrive" &&
|
|
r.sizeLocal === 0 &&
|
|
password === ""
|
|
) {
|
|
// special treatment for empty files for OneDrive
|
|
// TODO: it's ugly, any other way?
|
|
// special treatment for OneDrive: do nothing, skip empty file without encryption
|
|
// if it's empty folder, or it's encrypted file/folder, it continues to be uploaded.
|
|
} else {
|
|
const remoteObjMeta = await client.uploadToRemote(
|
|
r.key,
|
|
vault,
|
|
false,
|
|
password,
|
|
remoteEncryptedKey
|
|
);
|
|
await upsertSyncMetaMappingDataByVault(
|
|
client.serviceType,
|
|
db,
|
|
r.key,
|
|
r.mtimeLocal,
|
|
r.sizeLocal,
|
|
r.key,
|
|
remoteObjMeta.lastModified,
|
|
remoteObjMeta.size,
|
|
remoteObjMeta.etag,
|
|
vaultRandomID
|
|
);
|
|
}
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "downloadRemoteToLocal") {
|
|
await mkdirpInVault(r.key, vault); /* should be unnecessary */
|
|
await client.downloadFromRemote(
|
|
r.key,
|
|
vault,
|
|
r.mtimeRemote,
|
|
password,
|
|
remoteEncryptedKey
|
|
);
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "createFolder") {
|
|
if (!r.existLocal) {
|
|
await mkdirpInVault(r.key, vault);
|
|
}
|
|
if (!r.existRemote) {
|
|
const remoteObjMeta = await client.uploadToRemote(
|
|
r.key,
|
|
vault,
|
|
false,
|
|
password,
|
|
remoteEncryptedKey
|
|
);
|
|
await upsertSyncMetaMappingDataByVault(
|
|
client.serviceType,
|
|
db,
|
|
r.key,
|
|
r.mtimeLocal,
|
|
r.sizeLocal,
|
|
r.key,
|
|
remoteObjMeta.lastModified,
|
|
remoteObjMeta.size,
|
|
remoteObjMeta.etag,
|
|
vaultRandomID
|
|
);
|
|
}
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "uploadLocalDelHistToRemoteFolder") {
|
|
if (r.existLocal) {
|
|
await localDeleteFunc(r.key);
|
|
}
|
|
if (r.existRemote) {
|
|
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
|
|
}
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "keepRemoteDelHistFolder") {
|
|
if (r.existLocal) {
|
|
await localDeleteFunc(r.key);
|
|
}
|
|
if (r.existRemote) {
|
|
await client.deleteFromRemote(r.key, password, remoteEncryptedKey);
|
|
}
|
|
await clearDeleteRenameHistoryOfKeyAndVault(db, r.key, vaultRandomID);
|
|
} else if (r.decision === "skipFolder") {
|
|
// do nothing!
|
|
} else {
|
|
throw Error(`unknown decision in ${JSON.stringify(r)}`);
|
|
}
|
|
};
|
|
|
|
const splitThreeSteps = (syncPlan: SyncPlanType, sortedKeys: string[]) => {
|
|
const mixedStates = syncPlan.mixedStates;
|
|
const totalCount = sortedKeys.length || 0;
|
|
|
|
const folderCreationOps: FileOrFolderMixedState[][] = [];
|
|
const deletionOps: FileOrFolderMixedState[][] = [];
|
|
const uploadDownloads: FileOrFolderMixedState[][] = [];
|
|
let realTotalCount = 0;
|
|
|
|
for (let i = 0; i < sortedKeys.length; ++i) {
|
|
const key = sortedKeys[i];
|
|
const val: FileOrFolderMixedState = Object.assign({}, mixedStates[key]); // copy to avoid issue
|
|
|
|
if (val.decision === "skipFolder" || val.decision === "skipUploading") {
|
|
// pass
|
|
} else if (val.decision === "createFolder") {
|
|
const level = atWhichLevel(key);
|
|
if (folderCreationOps[level - 1] === undefined) {
|
|
folderCreationOps[level - 1] = [val];
|
|
} else {
|
|
folderCreationOps[level - 1].push(val);
|
|
}
|
|
realTotalCount += 1;
|
|
} else if (
|
|
val.decision === "uploadLocalDelHistToRemoteFolder" ||
|
|
val.decision === "keepRemoteDelHistFolder" ||
|
|
val.decision === "uploadLocalDelHistToRemote" ||
|
|
val.decision === "keepRemoteDelHist"
|
|
) {
|
|
const level = atWhichLevel(key);
|
|
if (deletionOps[level - 1] === undefined) {
|
|
deletionOps[level - 1] = [val];
|
|
} else {
|
|
deletionOps[level - 1].push(val);
|
|
}
|
|
realTotalCount += 1;
|
|
} else if (
|
|
val.decision === "uploadLocalToRemote" ||
|
|
val.decision === "downloadRemoteToLocal"
|
|
) {
|
|
if (uploadDownloads.length === 0) {
|
|
uploadDownloads[0] = [val];
|
|
} else {
|
|
uploadDownloads[0].push(val); // only one level needed here
|
|
}
|
|
realTotalCount += 1;
|
|
} else {
|
|
throw Error(`unknown decision ${val.decision} for ${key}`);
|
|
}
|
|
}
|
|
|
|
// the deletionOps should be run from max level to min level
|
|
// right now it is sorted by level from min to max (NOT length of key!)
|
|
// so we need to reverse it!
|
|
deletionOps.reverse(); // inplace reverse
|
|
|
|
return {
|
|
folderCreationOps: folderCreationOps,
|
|
deletionOps: deletionOps,
|
|
uploadDownloads: uploadDownloads,
|
|
realTotalCount: realTotalCount,
|
|
};
|
|
};
|
|
|
|
export const doActualSync = async (
|
|
client: RemoteClient,
|
|
db: InternalDBs,
|
|
vaultRandomID: string,
|
|
vault: Vault,
|
|
syncPlan: SyncPlanType,
|
|
sortedKeys: string[],
|
|
metadataFile: FileOrFolderMixedState,
|
|
origMetadata: MetadataOnRemote,
|
|
deletions: DeletionOnRemote[],
|
|
localDeleteFunc: any,
|
|
password: string = "",
|
|
concurrency: number = 1,
|
|
callbackSyncProcess?: any
|
|
) => {
|
|
const mixedStates = syncPlan.mixedStates;
|
|
const totalCount = sortedKeys.length || 0;
|
|
|
|
log.debug(`start syncing extra data firstly`);
|
|
await uploadExtraMeta(
|
|
client,
|
|
metadataFile,
|
|
origMetadata,
|
|
deletions,
|
|
password
|
|
);
|
|
log.debug(`finish syncing extra data firstly`);
|
|
|
|
log.debug(`concurrency === ${concurrency}`);
|
|
if (concurrency === 1) {
|
|
// run everything in sequence
|
|
// good old way
|
|
for (let i = 0; i < sortedKeys.length; ++i) {
|
|
const key = sortedKeys[i];
|
|
const val = mixedStates[key];
|
|
|
|
log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
|
|
|
|
if (callbackSyncProcess !== undefined) {
|
|
await callbackSyncProcess(i, totalCount, key, val.decision);
|
|
}
|
|
|
|
await dispatchOperationToActual(
|
|
key,
|
|
vaultRandomID,
|
|
val,
|
|
client,
|
|
db,
|
|
vault,
|
|
localDeleteFunc,
|
|
password
|
|
);
|
|
log.debug(`finished ${key}`);
|
|
}
|
|
|
|
return; // shortcut return, avoid too many nests below
|
|
}
|
|
|
|
const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } =
|
|
splitThreeSteps(syncPlan, sortedKeys);
|
|
const nested = [folderCreationOps, deletionOps, uploadDownloads];
|
|
const logTexts = [
|
|
`1. create all folders from shadowest to deepest, also check undefined decision`,
|
|
`2. delete files and folders from deepest to shadowest`,
|
|
`3. upload or download files in parallel, with the desired concurrency=${concurrency}`,
|
|
];
|
|
|
|
let realCounter = 0;
|
|
|
|
for (let i = 0; i < nested.length; ++i) {
|
|
log.debug(logTexts[i]);
|
|
|
|
const operations: FileOrFolderMixedState[][] = nested[i];
|
|
|
|
for (let j = 0; j < operations.length; ++j) {
|
|
const singleLevelOps: FileOrFolderMixedState[] | undefined =
|
|
operations[j];
|
|
|
|
if (singleLevelOps === undefined || singleLevelOps === null) {
|
|
continue;
|
|
}
|
|
|
|
const queue = new PQueue({ concurrency: concurrency, autoStart: true });
|
|
const potentialErrors: Error[] = [];
|
|
let tooManyErrors = false;
|
|
|
|
for (let k = 0; k < singleLevelOps.length; ++k) {
|
|
const val: FileOrFolderMixedState = singleLevelOps[k];
|
|
const key = val.key;
|
|
|
|
const fn = async () => {
|
|
log.debug(`start syncing "${key}" with plan ${JSON.stringify(val)}`);
|
|
|
|
if (callbackSyncProcess !== undefined) {
|
|
await callbackSyncProcess(
|
|
realCounter,
|
|
realTotalCount,
|
|
key,
|
|
val.decision
|
|
);
|
|
|
|
realCounter += 1;
|
|
}
|
|
|
|
await dispatchOperationToActual(
|
|
key,
|
|
vaultRandomID,
|
|
val,
|
|
client,
|
|
db,
|
|
vault,
|
|
localDeleteFunc,
|
|
password
|
|
);
|
|
|
|
log.debug(`finished ${key}`);
|
|
};
|
|
|
|
queue.add(fn).catch((e) => {
|
|
const msg = `${key}: ${e.message}`;
|
|
potentialErrors.push(new Error(msg));
|
|
if (potentialErrors.length >= 3) {
|
|
tooManyErrors = true;
|
|
queue.pause();
|
|
queue.clear();
|
|
}
|
|
});
|
|
}
|
|
|
|
await queue.onIdle();
|
|
|
|
if (potentialErrors.length > 0) {
|
|
if (tooManyErrors) {
|
|
potentialErrors.push(
|
|
new Error("too many errors, stop the remaining tasks")
|
|
);
|
|
}
|
|
throw new AggregateError(potentialErrors);
|
|
}
|
|
}
|
|
}
|
|
};
|