Compare commits

...

4 Commits

Author SHA1 Message Date
fyears d1e30e3536 bump to 0.4.21 2024-05-09 00:02:07 +08:00
fyears 36079fc1d0 add profiler 2024-05-09 00:01:30 +08:00
fyears 67467a5034 fail statusbar 2024-05-08 22:50:18 +08:00
fyears 2a3df8ab53 fix onedrive issue in enc 2024-05-08 22:37:34 +08:00
21 changed files with 294 additions and 38 deletions

View File

@ -1,7 +1,7 @@
{
"id": "remotely-save",
"name": "Remotely Save",
"version": "0.4.20",
"version": "0.4.21",
"minAppVersion": "0.13.21",
"description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
"author": "fyears",

View File

@ -1,7 +1,7 @@
{
"id": "remotely-save",
"name": "Remotely Save",
"version": "0.4.20",
"version": "0.4.21",
"minAppVersion": "0.13.21",
"description": "Yet another unofficial plugin allowing users to synchronize notes between local device and the cloud service.",
"author": "fyears",

View File

@ -1,6 +1,6 @@
{
"name": "remotely-save",
"version": "0.4.20",
"version": "0.4.21",
"description": "This is yet another sync plugin for Obsidian app.",
"scripts": {
"dev2": "node esbuild.config.mjs --watch",

View File

@ -107,6 +107,11 @@ export type CipherMethodType = "rclone-base64" | "openssl-base64" | "unknown";
export type QRExportType = "all_but_oauth2" | "dropbox" | "onedrive";
export interface ProfilerConfig {
enablePrinting?: boolean;
recordSize?: boolean;
}
export interface RemotelySavePluginSettings {
s3: S3Config;
webdav: WebdavConfig;
@ -141,6 +146,8 @@ export interface RemotelySavePluginSettings {
encryptionMethod?: CipherMethodType;
profiler?: ProfilerConfig;
/**
* @deprecated
*/

View File

@ -16,4 +16,5 @@ export abstract class FakeFs {
abstract checkConnect(callbackFunc?: any): Promise<boolean>;
abstract getUserDisplayName(): Promise<string>;
abstract revokeAuth(): Promise<any>;
abstract allowEmptyFile(): boolean;
}

View File

@ -736,4 +736,8 @@ export class FakeFsDropbox extends FakeFs {
return false;
}
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -273,9 +273,13 @@ export class FakeFsEncrypt extends FakeFs {
return copyEntityAndCopyKeyEncSizeEnc(innerEntity);
} else {
const now = Date.now();
let content = new ArrayBuffer(0);
if (!this.innerFs.allowEmptyFile()) {
content = new ArrayBuffer(1);
}
const innerEntity = await this.innerFs.writeFile(
keyEnc,
new ArrayBuffer(0),
content,
mtime ?? now,
ctime ?? now
);
@ -554,4 +558,8 @@ export class FakeFsEncrypt extends FakeFs {
async revokeAuth(): Promise<any> {
return await this.innerFs.revokeAuth();
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -170,4 +170,8 @@ export class FakeFsLocal extends FakeFs {
async revokeAuth(): Promise<any> {
throw new Error("Method not implemented.");
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -49,4 +49,8 @@ export class FakeFsMock extends FakeFs {
async revokeAuth(): Promise<any> {
throw new Error("Method not implemented.");
}
allowEmptyFile(): boolean {
throw new Error("Method not implemented.");
}
}

View File

@ -928,4 +928,8 @@ export class FakeFsOnedrive extends FakeFs {
async getRevokeAddr() {
return "https://account.live.com/consent/Manage";
}
allowEmptyFile(): boolean {
return false;
}
}

View File

@ -817,4 +817,8 @@ export class FakeFsS3 extends FakeFs {
async revokeAuth() {
throw new Error("Method not implemented.");
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -493,4 +493,8 @@ export class FakeFsWebdav extends FakeFs {
async revokeAuth() {
throw new Error("Method not implemented.");
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -239,4 +239,8 @@ export class FakeFsWebdis extends FakeFs {
async revokeAuth(): Promise<any> {
throw new Error("Method not implemented.");
}
allowEmptyFile(): boolean {
return true;
}
}

View File

@ -53,6 +53,7 @@
"statusbar_time_lessminute": "Synced last minute ago",
"statusbar_lastsync": "Synced {{time}} ago",
"statusbar_syncing": "Syncing...",
"statusbar_failed": "Last sync failed",
"statusbar_now": "Synced just now",
"statusbar_lastsync_label": "Last successful Sync on {{date}}",
"statusbar_lastsync_never": "Never Synced",
@ -329,6 +330,10 @@
"settings_profiler_results_desc": "The plugin records the time cost of each steps. Here you can export them to know which step is slow.",
"settings_profiler_results_notice": "Profiler results exported.",
"settings_profiler_results_button_all": "Export All",
"settings_profiler_enabledebugprint": "Enable Profiler Printing",
"settings_profiler_enabledebugprint_desc": "Print profiler result in each insertion to console or not?",
"settings_profiler_recordsize": "Enable Profiler Recording Size",
"settings_profiler_recordsize_desc": "Let profiler record object sizes or not?",
"settings_outputbasepathvaultid": "Output Vault Base Path And Randomly Assigned ID",
"settings_outputbasepathvaultid_desc": "For debugging purposes.",
"settings_outputbasepathvaultid_button": "Output",

View File

@ -53,6 +53,7 @@
"statusbar_time_lessminute": "一分钟之内同步",
"statusbar_lastsync": "上一次同步于:{{time}}",
"statusbar_syncing": "正在同步",
"statusbar_failed": "上次同步失败了",
"statusbar_now": "刚同步完",
"statusbar_lastsync_label": "上一次同步于:{{date}}",
"statusbar_lastsync_never": "没触发过同步",

View File

@ -52,6 +52,7 @@
"statusbar_time_lessminute": "一分鐘之內同步",
"statusbar_lastsync": "上一次同步於:{{time}}",
"statusbar_syncing": "正在同步",
"statusbar_failed": "上次同步失敗了",
"statusbar_now": "剛同步完",
"statusbar_lastsync_label": "上一次同步於:{{date}}",
"statusbar_lastsync_never": "沒觸發過同步",

View File

@ -62,7 +62,7 @@ import { getClient } from "./fsGetter";
import { FakeFsLocal } from "./fsLocal";
import { DEFAULT_WEBDIS_CONFIG } from "./fsWebdis";
import { changeMobileStatusBar } from "./misc";
import { Profiler } from "./profiler";
import { DEFAULT_PROFILER_CONFIG, Profiler } from "./profiler";
import { syncer } from "./sync";
const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
@ -96,6 +96,7 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
obfuscateSettingFile: true,
enableMobileStatusBar: false,
encryptionMethod: "unknown",
profiler: DEFAULT_PROFILER_CONFIG,
};
interface OAuth2Info {
@ -151,7 +152,11 @@ export default class RemotelySavePlugin extends Plugin {
appContainerObserver?: MutationObserver;
async syncRun(triggerSource: SyncTriggerSourceType = "manual") {
const profiler = new Profiler();
const profiler = new Profiler(
undefined,
this.settings.profiler?.enablePrinting ?? false,
this.settings.profiler?.recordSize ?? false
);
const fsLocal = new FakeFsLocal(
this.app.vault,
this.settings.syncConfigDir ?? false,
@ -337,11 +342,15 @@ export default class RemotelySavePlugin extends Plugin {
}
};
const statusBarFunc = async (s: SyncTriggerSourceType, step: number) => {
const statusBarFunc = async (
s: SyncTriggerSourceType,
step: number,
everythingOk: boolean
) => {
if (step === 1) {
// change status to "syncing..." on statusbar
this.updateLastSuccessSyncMsg(-1);
} else if (step === 8) {
} else if (step === 8 && everythingOk) {
const lastSuccessSyncMillis = Date.now();
await upsertLastSuccessSyncTimeByVault(
this.db,
@ -349,6 +358,8 @@ export default class RemotelySavePlugin extends Plugin {
lastSuccessSyncMillis
);
this.updateLastSuccessSyncMsg(lastSuccessSyncMillis);
} else if (!everythingOk) {
this.updateLastSuccessSyncMsg(-2); // magic number
}
};
@ -934,6 +945,16 @@ export default class RemotelySavePlugin extends Plugin {
}
}
if (this.settings.profiler === undefined) {
this.settings.profiler = DEFAULT_PROFILER_CONFIG;
}
if (this.settings.profiler.enablePrinting === undefined) {
this.settings.profiler.enablePrinting = false;
}
if (this.settings.profiler.recordSize === undefined) {
this.settings.profiler.recordSize = false;
}
await this.saveSettings();
}
@ -1266,6 +1287,11 @@ export default class RemotelySavePlugin extends Plugin {
lastSyncMsg = t("statusbar_syncing");
}
if (lastSuccessSyncMillis !== undefined && lastSuccessSyncMillis === -2) {
lastSyncMsg = t("statusbar_failed");
lastSyncLabelMsg = t("statusbar_failed");
}
if (lastSuccessSyncMillis !== undefined && lastSuccessSyncMillis > 0) {
const deltaTime = Date.now() - lastSuccessSyncMillis;

View File

@ -641,3 +641,41 @@ export const fixEntityListCasesInplace = (entities: { keyRaw: string }[]) => {
return entities;
};
/**
* https://stackoverflow.com/questions/1248302/how-to-get-the-size-of-a-javascript-object
* @param object
* @returns bytes
*/
export const roughSizeOfObject = (object: any) => {
const objectList: any[] = [];
const stack = [object];
let bytes = 0;
while (stack.length) {
const value = stack.pop();
switch (typeof value) {
case "boolean":
bytes += 4;
break;
case "string":
bytes += value.length * 2;
break;
case "number":
bytes += 8;
break;
case "object":
if (!objectList.includes(value)) {
objectList.push(value);
for (const prop in value) {
if (value.hasOwnProperty(prop)) {
stack.push(value[prop]);
}
}
}
break;
}
}
return bytes;
};

View File

@ -1,29 +1,43 @@
import type { SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import type { ProfilerConfig, SUPPORTED_SERVICES_TYPE } from "./baseTypes";
import { type InternalDBs, insertProfilerResultByVault } from "./localdb";
import { unixTimeToStr } from "./misc";
import { roughSizeOfObject, unixTimeToStr } from "./misc";
interface BreakPoint {
label: string;
fakeTimeMilli: number; // it's NOT a unix timestamp
indent: number;
size?: number;
}
export const DEFAULT_PROFILER_CONFIG: ProfilerConfig = {
enablePrinting: false,
recordSize: false,
};
export class Profiler {
startTime: number;
breakPoints: BreakPoint[];
indent: number;
constructor(label?: string) {
enablePrinting: boolean;
recordSize: boolean;
constructor(label?: string, enablePrinting?: boolean, recordSize?: boolean) {
this.breakPoints = [];
this.indent = 0;
this.startTime = 0;
this.enablePrinting = enablePrinting ?? false;
this.recordSize = recordSize ?? false;
if (label !== undefined) {
this.startTime = Date.now();
this.breakPoints.push({
const p = {
label: label,
fakeTimeMilli: performance.now(),
indent: this.indent,
});
};
this.breakPoints.push(p);
if (this.enablePrinting) {
console.debug(this.toString(-1));
}
}
}
@ -31,11 +45,36 @@ export class Profiler {
if (this.breakPoints.length === 0) {
this.startTime = Date.now();
}
this.breakPoints.push({
const p = {
label: label,
fakeTimeMilli: performance.now(),
indent: this.indent,
});
};
this.breakPoints.push(p);
if (this.enablePrinting) {
console.debug(this.toString(-1));
}
return this;
}
insertSize(label: string, obj: any) {
if (!this.recordSize) {
return;
}
if (this.breakPoints.length === 0) {
this.startTime = Date.now();
}
const p = {
label: label,
fakeTimeMilli: performance.now(),
indent: this.indent,
size: roughSizeOfObject(obj),
};
this.breakPoints.push(p);
if (this.enablePrinting) {
console.debug(this.toString(-1));
}
return this;
}
@ -57,7 +96,31 @@ export class Profiler {
return this;
}
toString() {
toString(idx?: number) {
if (idx !== undefined) {
let i = idx;
if (idx < 0) {
i = this.breakPoints.length + idx;
}
const label = this.breakPoints?.[i]["label"];
const indent = this.breakPoints?.[i]["indent"];
let millsec = 0;
if (i >= 1) {
millsec =
Math.round(
(this.breakPoints?.[i]["fakeTimeMilli"] -
this.breakPoints?.[i - 1]["fakeTimeMilli"]) *
10
) / 10.0;
}
let res = `${" ".repeat(indent)}[${label}]: ${millsec}ms`;
if (this.breakPoints[i].hasOwnProperty("size")) {
const size = this.breakPoints[i].size as number;
res += `, size=${size}`;
}
return res;
}
if (this.breakPoints.length === 0) {
return "nothing in profiler";
}
@ -67,15 +130,7 @@ export class Profiler {
if (i === 0) {
res += `\n[${this.breakPoints[i]["label"]}]: start`;
} else {
const label = this.breakPoints[i]["label"];
const indent = this.breakPoints[i]["indent"];
const millsec =
Math.round(
(this.breakPoints[i]["fakeTimeMilli"] -
this.breakPoints[i - 1]["fakeTimeMilli"]) *
10
) / 10.0;
res += `\n${" ".repeat(indent)}[${label}]: ${millsec}ms`;
res += `\n${this.toString(i)}`;
}
}

View File

@ -57,6 +57,7 @@ import {
checkHasSpecialCharForDir,
stringToFragment,
} from "./misc";
import { DEFAULT_PROFILER_CONFIG } from "./profiler";
class PasswordModal extends Modal {
plugin: RemotelySavePlugin;
@ -2478,6 +2479,44 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
});
});
new Setting(debugDiv)
.setName(t("settings_profiler_enabledebugprint"))
.setDesc(t("settings_profiler_enabledebugprint_desc"))
.addDropdown((dropdown) => {
dropdown.addOption("enable", t("enable"));
dropdown.addOption("disable", t("disable"));
dropdown
.setValue(
this.plugin.settings.profiler?.enablePrinting ? "enable" : "disable"
)
.onChange(async (val: string) => {
if (this.plugin.settings.profiler === undefined) {
this.plugin.settings.profiler = DEFAULT_PROFILER_CONFIG;
}
this.plugin.settings.profiler.enablePrinting = val === "enable";
await this.plugin.saveSettings();
});
});
new Setting(debugDiv)
.setName(t("settings_profiler_recordsize"))
.setDesc(t("settings_profiler_recordsize_desc"))
.addDropdown((dropdown) => {
dropdown.addOption("enable", t("enable"));
dropdown.addOption("disable", t("disable"));
dropdown
.setValue(
this.plugin.settings.profiler?.recordSize ? "enable" : "disable"
)
.onChange(async (val: string) => {
if (this.plugin.settings.profiler === undefined) {
this.plugin.settings.profiler = DEFAULT_PROFILER_CONFIG;
}
this.plugin.settings.profiler.recordSize = val === "enable";
await this.plugin.saveSettings();
});
});
new Setting(debugDiv)
.setName(t("settings_outputbasepathvaultid"))
.setDesc(t("settings_outputbasepathvaultid_desc"))

View File

@ -30,6 +30,7 @@ import {
getParentFolder,
isHiddenPath,
isSpecialFolderNameToSkip,
roughSizeOfObject,
unixTimeToStr,
} from "./misc";
import type { Profiler } from "./profiler";
@ -158,6 +159,9 @@ const ensembleMixedEnties = async (
): Promise<SyncPlanType> => {
profiler.addIndent();
profiler.insert("ensembleMixedEnties: enter");
profiler.insertSize("sizeof localEntityList", localEntityList);
profiler.insertSize("sizeof prevSyncEntityList", prevSyncEntityList);
profiler.insertSize("sizeof remoteEntityList", remoteEntityList);
const finalMappings: SyncPlanType = {};
@ -187,6 +191,7 @@ const ensembleMixedEnties = async (
}
profiler.insert("ensembleMixedEnties: finish remote");
profiler.insertSize("sizeof finalMappings", finalMappings);
if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
// Special checking:
@ -227,6 +232,7 @@ const ensembleMixedEnties = async (
}
profiler.insert("ensembleMixedEnties: finish prevSync");
profiler.insertSize("sizeof finalMappings", finalMappings);
// local has to be last
// because we want to get keyEnc based on the remote
@ -260,6 +266,7 @@ const ensembleMixedEnties = async (
}
profiler.insert("ensembleMixedEnties: finish local");
profiler.insertSize("sizeof finalMappings", finalMappings);
// console.debug("in the end of ensembleMixedEnties, finalMappings is:");
// console.debug(finalMappings);
@ -280,7 +287,9 @@ const getSyncPlanInplace = async (
skipSizeLargerThan: number,
conflictAction: ConflictActionType,
syncDirection: SyncDirectionType,
profiler: Profiler
profiler: Profiler,
settings: RemotelySavePluginSettings,
triggerSource: SyncTriggerSourceType
) => {
profiler.addIndent();
profiler.insert("getSyncPlanInplace: enter");
@ -289,10 +298,17 @@ const getSyncPlanInplace = async (
(k1, k2) => k2.length - k1.length
);
profiler.insert("getSyncPlanInplace: finish sorting");
profiler.insertSize("sizeof sortedKeys", sortedKeys);
const keptFolder = new Set<string>();
for (let i = 0; i < sortedKeys.length; ++i) {
if (i % 100 === 0) {
profiler.insertSize(
`sizeof sortedKeys in the beginning of i=${i}`,
mixedEntityMappings
);
}
const key = sortedKeys[i];
const mixedEntry = mixedEntityMappings[key];
const { local, prevSync, remote } = mixedEntry;
@ -703,16 +719,28 @@ const getSyncPlanInplace = async (
const currTimeFmt = unixTimeToStr(currTime);
// because the path should not as / in the beginning,
// we should be safe to add these keys:
const sizeofmixedEntityMappings = roughSizeOfObject(mixedEntityMappings);
mixedEntityMappings["/$@meta"] = {
key: "/$@meta", // don't mess up with the types
sideNotes: {
version: "2024047 fs version",
version: "20240508 fs version",
generateTime: currTime,
generateTimeFmt: currTimeFmt,
service: settings.serviceType,
hasPassword: settings.password !== "",
syncConfigDir: settings.syncConfigDir,
conflictAction: conflictAction,
syncDirection: syncDirection,
triggerSource: triggerSource,
sizeof: sizeofmixedEntityMappings
},
};
profiler.insert("getSyncPlanInplace: exit");
profiler.insertSize(
"sizeof mixedEntityMappings in the end of getSyncPlanInplace",
mixedEntityMappings
);
profiler.removeIndent();
return mixedEntityMappings;
@ -1126,6 +1154,16 @@ export const doActualSync = async (
console.debug(`realTotalCount: ${realTotalCount}`);
profiler.insert("doActualSync: finish splitting steps");
profiler.insertSize(
"doActualSync: sizeof onlyMarkSyncedOps",
onlyMarkSyncedOps
);
profiler.insertSize(
"doActualSync: sizeof folderCreationOps",
folderCreationOps
);
profiler.insertSize("doActualSync: sizeof realTotalCount", deletionOps);
console.debug(`protectModifyPercentage: ${protectModifyPercentage}`);
if (
@ -1283,26 +1321,32 @@ export async function syncer(
notifyFunc?: (s: SyncTriggerSourceType, step: number) => Promise<any>,
errNotifyFunc?: (s: SyncTriggerSourceType, error: Error) => Promise<any>,
ribboonFunc?: (s: SyncTriggerSourceType, step: number) => Promise<any>,
statusBarFunc?: (s: SyncTriggerSourceType, step: number) => any,
statusBarFunc?: (
s: SyncTriggerSourceType,
step: number,
everythingOk: boolean
) => any,
callbackSyncProcess?: any
) {
console.info(`startting sync.`);
markIsSyncingFunc(true);
let everythingOk = true;
let step = 0; // dry mode only
await notifyFunc?.(triggerSource, step);
step = 1;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
profiler.insert("start big sync func");
try {
step = 2;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
if (fsEncrypt.innerFs !== fsRemote) {
throw Error(`your enc should has inner of the remote`);
}
@ -1317,7 +1361,7 @@ export async function syncer(
step = 3;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
const remoteEntityList = await fsEncrypt.walk();
// console.debug(`remoteEntityList:`);
// console.debug(remoteEntityList);
@ -1326,7 +1370,7 @@ export async function syncer(
step = 4;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
const localEntityList = await fsLocal.walk();
// console.debug(`localEntityList:`);
// console.debug(localEntityList);
@ -1335,7 +1379,7 @@ export async function syncer(
step = 5;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
const prevSyncEntityList = await getAllPrevSyncRecordsByVaultAndProfile(
db,
vaultRandomID,
@ -1348,7 +1392,7 @@ export async function syncer(
step = 6;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
let mixedEntityMappings = await ensembleMixedEnties(
localEntityList,
prevSyncEntityList,
@ -1369,7 +1413,9 @@ export async function syncer(
settings.skipSizeLargerThan ?? -1,
settings.conflictAction ?? "keep_newer",
settings.syncDirection ?? "bidirectional",
profiler
profiler,
settings,
triggerSource
);
console.debug(`mixedEntityMappings:`);
console.debug(mixedEntityMappings); // for debugging
@ -1391,7 +1437,7 @@ export async function syncer(
if (triggerSource !== "dry") {
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
await doActualSync(
mixedEntityMappings,
fsLocal,
@ -1409,13 +1455,14 @@ export async function syncer(
} else {
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
profiler.insert(
`finish step${step} (skip actual sync because of dry run)`
);
}
} catch (error: any) {
profiler.insert("start error branch");
everythingOk = false;
await errNotifyFunc?.(triggerSource, error as Error);
profiler.insert("finish error branch");
@ -1429,7 +1476,7 @@ export async function syncer(
step = 8;
await notifyFunc?.(triggerSource, step);
await ribboonFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step);
await statusBarFunc?.(triggerSource, step, everythingOk);
console.info(`endding sync.`);
markIsSyncingFunc(false);