Mirror
/
remotely-save
mirror of https://github.com/remotely-save/remotely-save.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix for encryption

This commit is contained in:
fyears 2024-02-25 15:14:28 +08:00
parent 40020c3e44
commit 1f4737bfb8
9 changed files with 182 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/baseTypes.ts
View File

@ -169,8 +169,9 @@ export type DecisionTypeForMixedEntity =
* everything should be flat and primitive, so that we can copy.
*/
export interface Entity {
key: string;
keyEnc: string;
key?: string;
keyEnc?: string;
keyRaw: string;
mtimeCli?: number;
mtimeCliFmt?: string;
mtimeSvr?: number;
@ -178,7 +179,8 @@ export interface Entity {
prevSyncTime?: number;
prevSyncTimeFmt?: string;
size?: number; // might be unknown or to be filled
sizeEnc: number;
sizeEnc?: number;
sizeRaw: number;
hash?: string;
etag?: string;
}

12
src/local.ts
View File

@ -32,20 +32,20 @@ export const getLocalEntityList = async (
);
}
r = {
key: entry.path,
keyEnc: entry.path,
key: entry.path, // local always unencrypted
keyRaw: entry.path,
mtimeCli: mtimeLocal,
mtimeSvr: mtimeLocal,
size: entry.stat.size,
sizeEnc: entry.stat.size,
size: entry.stat.size, // local always unencrypted
sizeRaw: entry.stat.size,
};
} else if (entry instanceof TFolder) {
key = `${entry.path}/`;
r = {
key: key,
keyEnc: key,
keyRaw: key,
size: 0,
sizeEnc: 0,
sizeRaw: 0,
};
} else {
throw Error(`unexpected ${entry}`);

4
src/main.ts
View File

@ -240,6 +240,7 @@ export default class RemotelySavePlugin extends Plugin {
() => self.saveSettings()
);
const remoteEntityList = await client.listAllFromRemote();
log.debug("remoteEntityList:");
log.debug(remoteEntityList);
if (this.settings.currLogLevel === "info") {
@ -269,6 +270,7 @@ export default class RemotelySavePlugin extends Plugin {
this.app.vault.configDir,
this.manifest.id
);
log.debug("localEntityList:");
log.debug(localEntityList);
if (this.settings.currLogLevel === "info") {
@ -281,6 +283,7 @@ export default class RemotelySavePlugin extends Plugin {
this.db,
this.vaultRandomID
);
log.debug("prevSyncEntityList:");
log.debug(prevSyncEntityList);
if (this.settings.currLogLevel === "info") {
@ -305,6 +308,7 @@ export default class RemotelySavePlugin extends Plugin {
this.settings.skipSizeLargerThan ?? -1,
this.settings.conflictAction ?? "keep_newer"
);
log.info(`mixedEntityMappings:`);
log.info(mixedEntityMappings); // for debugging
await insertSyncPlanRecordByVault(
this.db,

8
src/obsFolderLister.ts
View File

@ -79,12 +79,12 @@ export const listFilesInObsFolder = async (
return {
itself: {
key: isFolder ? `${x}/` : x,
keyEnc: isFolder ? `${x}/` : x,
key: isFolder ? `${x}/` : x, // local always unencrypted
keyRaw: isFolder ? `${x}/` : x,
mtimeCli: statRes.mtime,
mtimeSvr: statRes.mtime,
size: statRes.size,
sizeEnc: statRes.size,
size: statRes.size, // local always unencrypted
sizeRaw: statRes.size,
},
children: children,
};

17
src/remoteForDropbox.ts
View File

@ -83,22 +83,18 @@ const fromDropboxItemToEntity = (
if (x[".tag"] === "folder") {
return {
key: key,
keyEnc: key,
size: 0,
sizeEnc: 0,
keyRaw: key,
sizeRaw: 0,
etag: `${x.id}\t`,
} as Entity;
} else if (x[".tag"] === "file") {
const mtimeCli = Date.parse(x.client_modified).valueOf();
const mtimeSvr = Date.parse(x.server_modified).valueOf();
return {
key: key,
keyEnc: key,
keyRaw: key,
mtimeCli: mtimeCli,
mtimeSvr: mtimeSvr,
size: x.size,
sizeEnc: x.size,
sizeRaw: x.size,
hash: x.content_hash,
etag: `${x.id}\t${x.content_hash}`,
} as Entity;
@ -469,6 +465,11 @@ export const uploadToRemote = async (
let uploadFile = fileOrFolderPath;
if (password !== "") {
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
throw Error(
`uploadToRemote(dropbox) you have password but remoteEncryptedKey is empty!`
);
}
uploadFile = remoteEncryptedKey;
}
uploadFile = getDropboxPath(uploadFile, client.remoteBaseDir);

11
src/remoteForOnedrive.ts
View File

@ -351,12 +351,10 @@ const fromDriveItemToEntity = (x: DriveItem, remoteBaseDir: string): Entity => {
const mtimeSvr = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
const mtimeCli = Date.parse(x?.fileSystemInfo!.lastModifiedDateTime!);
return {
key: key,
keyEnc: key,
keyRaw: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeCli,
size: isFolder ? 0 : x.size!,
sizeEnc: isFolder ? 0 : x.size!,
sizeRaw: isFolder ? 0 : x.size!,
// hash: ?? // TODO
etag: x.cTag || "", // do NOT use x.eTag because it changes if meta changes
};
@ -708,6 +706,11 @@ export const uploadToRemote = async (
let uploadFile = fileOrFolderPath;
if (password !== "") {
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
throw Error(
`uploadToRemote(onedrive) you have password but remoteEncryptedKey is empty!`
);
}
uploadFile = remoteEncryptedKey;
}
uploadFile = getOnedrivePath(uploadFile, client.remoteBaseDir);

32
src/remoteForS3.ts
View File

@ -238,12 +238,10 @@ const fromS3ObjectToEntity = (
}
const key = getLocalNoPrefixPath(x.Key!, remotePrefix);
const r: Entity = {
key: key,
keyEnc: key,
keyRaw: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeCli,
size: x.Size!,
sizeEnc: x.Size!,
sizeRaw: x.Size!,
etag: x.ETag,
};
return r;
@ -266,11 +264,21 @@ const fromS3HeadObjectToEntity = (
mtimeCli = m2;
}
}
// log.debug(
// `fromS3HeadObjectToEntity, fileOrFolderPathWithRemotePrefix=${fileOrFolderPathWithRemotePrefix}, remotePrefix=${remotePrefix}, x=${JSON.stringify(
// x
// )} `
// );
const key = getLocalNoPrefixPath(
fileOrFolderPathWithRemotePrefix,
remotePrefix
);
// log.debug(`fromS3HeadObjectToEntity, key=${key} after removing prefix`);
return {
key: getLocalNoPrefixPath(fileOrFolderPathWithRemotePrefix, remotePrefix),
keyRaw: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeCli,
size: x.ContentLength,
sizeRaw: x.ContentLength,
etag: x.ETag,
} as Entity;
};
@ -361,9 +369,15 @@ export const uploadToRemote = async (
log.debug(`uploading ${fileOrFolderPath}`);
let uploadFile = fileOrFolderPath;
if (password !== "") {
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
throw Error(
`uploadToRemote(s3) you have password but remoteEncryptedKey is empty!`
);
}
uploadFile = remoteEncryptedKey;
}
uploadFile = getRemoteWithPrefixPath(uploadFile, s3Config.remotePrefix ?? "");
// log.debug(`actual uploadFile=${uploadFile}`);
const isFolder = fileOrFolderPath.endsWith("/");
if (isFolder && isRecursively) {
@ -459,9 +473,9 @@ export const uploadToRemote = async (
await upload.done();
const res = await getRemoteMeta(s3Client, s3Config, uploadFile);
log.debug(
`uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}`
);
// log.debug(
// `uploaded ${uploadFile} with res=${JSON.stringify(res, null, 2)}`
// );
return res;
}
};

11
src/remoteForWebdav.ts
View File

@ -212,12 +212,10 @@ const fromWebdavItemToEntity = (x: FileStat, remoteBaseDir: string) => {
}
const mtimeSvr = Date.parse(x.lastmod).valueOf();
return {
key: key,
keyEnc: key,
keyRaw: key,
mtimeSvr: mtimeSvr,
mtimeCli: mtimeSvr, // no universal way to set mtime in webdav
size: x.size,
sizeEnc: x.size,
sizeRaw: x.size,
etag: x.etag,
} as Entity;
};
@ -346,6 +344,11 @@ export const uploadToRemote = async (
await client.init();
let uploadFile = fileOrFolderPath;
if (password !== "") {
if (remoteEncryptedKey === undefined || remoteEncryptedKey === "") {
throw Error(
`uploadToRemote(webdav) you have password but remoteEncryptedKey is empty!`
);
}
uploadFile = remoteEncryptedKey;
}
uploadFile = getWebdavPath(uploadFile, client.remoteBaseDir);

186
src/sync.ts
View File

@ -75,7 +75,7 @@ export const isPasswordOk = async (
reason: "empty_remote",
};
}
const santyCheckKey = remote[0].key;
const santyCheckKey = remote[0].keyRaw;
if (santyCheckKey.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
// this is encrypted using old base32!
// try to decrypt it using the provided password.
@ -161,6 +161,9 @@ const isSkipItemByName = (
configDir: string,
ignorePaths: string[]
) => {
if (key === undefined) {
throw Error(`isSkipItemByName meets undefinded key!`);
}
if (ignorePaths !== undefined && ignorePaths.length > 0) {
for (const r of ignorePaths) {
if (XRegExp(r, "A").test(key)) {
@ -218,17 +221,25 @@ const copyEntityAndFixTimeFormat = (src: Entity) => {
*/
const decryptRemoteEntityInplace = async (remote: Entity, password: string) => {
if (password == undefined || password === "") {
remote.key = remote.keyEnc;
remote.size = remote.sizeEnc;
remote.key = remote.keyRaw;
remote.keyEnc = remote.keyRaw;
remote.size = remote.sizeRaw;
remote.sizeEnc = remote.sizeRaw;
return remote;
}
if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
if (remote.keyRaw.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE32)) {
remote.keyEnc = remote.keyRaw;
remote.key = await decryptBase32ToString(remote.keyEnc, password);
} else if (remote.keyEnc.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
remote.sizeEnc = remote.sizeRaw;
} else if (remote.keyRaw.startsWith(MAGIC_ENCRYPTED_PREFIX_BASE64URL)) {
remote.keyEnc = remote.keyRaw;
remote.key = await decryptBase64urlToString(remote.keyEnc, password);
remote.sizeEnc = remote.sizeRaw;
} else {
throw Error(`unexpected key to decrypt=${remote.keyEnc}`);
throw Error(
`unexpected key to decrypt: ${JSON.stringify(remote, null, 2)}`
);
}
// TODO
@ -245,7 +256,7 @@ const decryptRemoteEntityInplace = async (remote: Entity, password: string) => {
*/
const ensureMTimeOfRemoteEntityValid = (remote: Entity) => {
if (
!remote.key.endsWith("/") &&
!remote.key!.endsWith("/") &&
remote.mtimeCli === undefined &&
remote.mtimeSvr === undefined
) {
@ -273,19 +284,36 @@ const encryptLocalEntityInplace = async (
password: string,
remoteKeyEnc: string | undefined
) => {
if (password == undefined || password === "") {
local.sizeEnc = local.size!; // if no enc, the remote file has the same size
local.keyEnc = local.key;
// log.debug(
// `encryptLocalEntityInplace: local=${JSON.stringify(
// local,
// null,
// 2
// )}, password=${
// password === undefined || password === "" ? "[empty]" : "[not empty]"
// }, remoteKeyEnc=${remoteKeyEnc}`
// );
if (local.key === undefined) {
// local.key should always have value
throw Error(`local ${local.keyRaw} is abnormal without key`);
}
if (password === undefined || password === "") {
local.sizeEnc = local.sizeRaw; // if no enc, the remote file has the same size
local.keyEnc = local.keyRaw;
return local;
}
// below is for having password
if (local.size === local.sizeEnc) {
// size not transformed yet, we need to compute sizeEnc
if (local.sizeEnc === undefined && local.size !== undefined) {
// it's not filled yet, we fill it
// local.size is possibly undefined if it's "prevSync" Entity
// but local.key should always have value
local.sizeEnc = getSizeFromOrigToEnc(local.size);
}
if (local.key === local.keyEnc) {
if (local.keyEnc === undefined || local.keyEnc === "") {
if (
remoteKeyEnc !== undefined &&
remoteKeyEnc !== "" &&
@ -328,7 +356,7 @@ export const ensembleMixedEnties = async (
)
);
const key = remoteCopied.key;
const key = remoteCopied.key!;
if (
isSkipItemByName(
key,
@ -347,37 +375,47 @@ export const ensembleMixedEnties = async (
};
}
for (const prevSync of prevSyncEntityList) {
const key = prevSync.key;
if (
isSkipItemByName(
key,
syncConfigDir,
syncUnderscoreItems,
configDir,
ignorePaths
)
) {
continue;
}
if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
// Special checking:
// if one side is totally empty,
// usually that's a hard rest.
// So we need to ignore everything of prevSyncEntityList to avoid deletions!
// TODO: acutally erase everything of prevSyncEntityList?
// TODO: local should also go through a isSkipItemByName checking beforehand
} else {
// normally go through the prevSyncEntityList
for (const prevSync of prevSyncEntityList) {
const key = prevSync.key!;
if (
isSkipItemByName(
key,
syncConfigDir,
syncUnderscoreItems,
configDir,
ignorePaths
)
) {
continue;
}
if (finalMappings.hasOwnProperty(key)) {
const prevSyncCopied = await encryptLocalEntityInplace(
copyEntityAndFixTimeFormat(prevSync),
password,
finalMappings[key].remote?.keyEnc
);
finalMappings[key].prevSync = prevSyncCopied;
} else {
const prevSyncCopied = await encryptLocalEntityInplace(
copyEntityAndFixTimeFormat(prevSync),
password,
undefined
);
finalMappings[key] = {
key: key,
prevSync: prevSyncCopied,
};
if (finalMappings.hasOwnProperty(key)) {
const prevSyncCopied = await encryptLocalEntityInplace(
copyEntityAndFixTimeFormat(prevSync),
password,
finalMappings[key].remote?.keyEnc
);
finalMappings[key].prevSync = prevSyncCopied;
} else {
const prevSyncCopied = await encryptLocalEntityInplace(
copyEntityAndFixTimeFormat(prevSync),
password,
undefined
);
finalMappings[key] = {
key: key,
prevSync: prevSyncCopied,
};
}
}
}
@ -385,7 +423,7 @@ export const ensembleMixedEnties = async (
// because we want to get keyEnc based on the remote
// (we don't consume prevSync here because it gains no benefit)
for (const local of localEntityList) {
const key = local.key;
const key = local.key!;
if (
isSkipItemByName(
key,
@ -514,7 +552,7 @@ export const getSyncPlanInplace = async (
// If only one compares true (no prev also means it compares False), the other is modified. Backup and sync.
if (
skipSizeLargerThan <= 0 ||
remote.sizeEnc <= skipSizeLargerThan
remote.sizeEnc! <= skipSizeLargerThan
) {
mixedEntry.decisionBranch = 9;
mixedEntry.decision = "modified_remote";
@ -530,7 +568,7 @@ export const getSyncPlanInplace = async (
// If only one compares true (no prev also means it compares False), the other is modified. Backup and sync.
if (
skipSizeLargerThan <= 0 ||
local.sizeEnc <= skipSizeLargerThan
local.sizeEnc! <= skipSizeLargerThan
) {
mixedEntry.decisionBranch = 10;
mixedEntry.decision = "modified_local";
@ -559,7 +597,7 @@ export const getSyncPlanInplace = async (
keptFolder.add(getParentFolder(key));
}
} else if (conflictAction === "keep_larger") {
if (local.sizeEnc >= remote.sizeEnc) {
if (local.sizeEnc! >= remote.sizeEnc!) {
mixedEntry.decisionBranch = 13;
mixedEntry.decision = "conflict_created_keep_local";
keptFolder.add(getParentFolder(key));
@ -588,7 +626,7 @@ export const getSyncPlanInplace = async (
keptFolder.add(getParentFolder(key));
}
} else if (conflictAction === "keep_larger") {
if (local.sizeEnc >= remote.sizeEnc) {
if (local.sizeEnc! >= remote.sizeEnc!) {
mixedEntry.decisionBranch = 18;
mixedEntry.decision = "conflict_modified_keep_local";
keptFolder.add(getParentFolder(key));
@ -616,7 +654,10 @@ export const getSyncPlanInplace = async (
// A is missing
if (prevSync === undefined) {
// if B is not in the previous list, B is new
if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) {
if (
skipSizeLargerThan <= 0 ||
remote.sizeEnc! <= skipSizeLargerThan
) {
mixedEntry.decisionBranch = 3;
mixedEntry.decision = "created_remote";
keptFolder.add(getParentFolder(key));
@ -637,7 +678,10 @@ export const getSyncPlanInplace = async (
mixedEntry.decision = "deleted_local";
} else {
// if B is in the previous list and MODIFIED, B has been deleted by A but modified by B
if (skipSizeLargerThan <= 0 || remote.sizeEnc <= skipSizeLargerThan) {
if (
skipSizeLargerThan <= 0 ||
remote.sizeEnc! <= skipSizeLargerThan
) {
mixedEntry.decisionBranch = 5;
mixedEntry.decision = "modified_remote";
keptFolder.add(getParentFolder(key));
@ -654,7 +698,7 @@ export const getSyncPlanInplace = async (
if (prevSync === undefined) {
// if A is not in the previous list, A is new
if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) {
if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) {
mixedEntry.decisionBranch = 6;
mixedEntry.decision = "created_local";
keptFolder.add(getParentFolder(key));
@ -675,7 +719,7 @@ export const getSyncPlanInplace = async (
mixedEntry.decision = "deleted_remote";
} else {
// if A is in the previous list and MODIFIED, A has been deleted by B but modified by A
if (skipSizeLargerThan <= 0 || local.sizeEnc <= skipSizeLargerThan) {
if (skipSizeLargerThan <= 0 || local.sizeEnc! <= skipSizeLargerThan) {
mixedEntry.decisionBranch = 8;
mixedEntry.decision = "modified_local";
keptFolder.add(getParentFolder(key));
@ -744,9 +788,9 @@ const splitThreeStepsOnEntityMappings = (
val.decision === "folder_existed_remote" ||
val.decision === "folder_to_be_created"
) {
log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`);
// log.debug(`splitting folder: key=${key},val=${JSON.stringify(val)}`);
const level = atWhichLevel(key);
log.debug(`atWhichLevel: ${level}`);
// log.debug(`atWhichLevel: ${level}`);
const k = folderCreationOps[level - 1];
if (k === undefined || k === null) {
folderCreationOps[level - 1] = [val];
@ -818,13 +862,13 @@ const dispatchOperationToActualV3 = async (
localDeleteFunc: any,
password: string
) => {
log.debug(
`inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify(
r,
null,
2
)}`
);
// log.debug(
// `inside dispatchOperationToActualV3, key=${key}, r=${JSON.stringify(
// r,
// null,
// 2
// )}`
// );
if (r.decision === "only_history") {
clearPrevSyncRecordByVault(db, vaultRandomID, key);
} else if (
@ -850,6 +894,7 @@ const dispatchOperationToActualV3 = async (
// special treatment for OneDrive: do nothing, skip empty file without encryption
// if it's empty folder, or it's encrypted file/folder, it continues to be uploaded.
} else {
// log.debug(`before upload in sync, r=${JSON.stringify(r, null, 2)}`);
const remoteObjMeta = await client.uploadToRemote(
r.key,
vault,
@ -857,6 +902,7 @@ const dispatchOperationToActualV3 = async (
password,
r.local!.keyEnc
);
await decryptRemoteEntityInplace(remoteObjMeta, password);
await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta);
}
} else if (
@ -897,6 +943,8 @@ const dispatchOperationToActualV3 = async (
password,
r.local!.keyEnc
);
// we need to decrypt the key!!!
await decryptRemoteEntityInplace(remoteObjMeta, password);
await upsertPrevSyncRecordByVault(db, vaultRandomID, remoteObjMeta);
} else if (r.decision === "folder_to_be_deleted") {
await localDeleteFunc(r.key);
@ -921,10 +969,10 @@ export const doActualSync = async (
log.debug(`concurrency === ${concurrency}`);
const { folderCreationOps, deletionOps, uploadDownloads, realTotalCount } =
splitThreeStepsOnEntityMappings(mixedEntityMappings);
log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`);
log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`);
log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`);
log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`);
// log.debug(`folderCreationOps: ${JSON.stringify(folderCreationOps)}`);
// log.debug(`deletionOps: ${JSON.stringify(deletionOps)}`);
// log.debug(`uploadDownloads: ${JSON.stringify(uploadDownloads)}`);
// log.debug(`realTotalCount: ${JSON.stringify(realTotalCount)}`);
const nested = [folderCreationOps, deletionOps, uploadDownloads];
const logTexts = [
@ -938,11 +986,11 @@ export const doActualSync = async (
log.debug(logTexts[i]);
const operations = nested[i];
log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`);
// log.debug(`curr operations=${JSON.stringify(operations, null, 2)}`);
for (let j = 0; j < operations.length; ++j) {
const singleLevelOps = operations[j];
log.debug(`singleLevelOps=${singleLevelOps}`);
log.debug(`singleLevelOps=${JSON.stringify(singleLevelOps, null, 2)}`);
if (singleLevelOps === undefined || singleLevelOps === null) {
continue;
}