Squashed commit of sync hidden files:

commit 73967756d51d246b3ca203aa3683cdfebf567000
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Mar 13 22:41:28 2022 +0800

    fix typo

commit 08e16faa9a9ace9bec71acb723e0d70ca361d49f
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Mar 13 22:41:01 2022 +0800

    add modal in settings

commit 9db7194fa28cb62b1fa4c01ac7f746d5ac3b86a8
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Mar 13 22:03:00 2022 +0800

    working sync for .obsidian and _

commit 4be24ba092181c1c3e1aabe5e9d4e9fcff28f987
Author: fyears <1142836+fyears@users.noreply.github.com>
Date:   Sun Mar 13 16:07:10 2022 +0800

    more logic for hidden path

src/baseTypes.ts

@@ -67,6 +67,8 @@ export interface RemotelySavePluginSettings {
   initRunAfterMilliseconds?: number;
   agreeToUploadExtraMetadata?: boolean;
   concurrency?: number;
+  syncConfigDir?: boolean;
+  syncUnderscoreItems?: boolean;
 }
 
 export interface RemoteItem {

src/main.ts

@@ -37,6 +37,7 @@ import { RemotelySaveSettingTab } from "./settings";
 import { fetchMetadataFile, parseRemoteItems, SyncStatusType } from "./sync";
 import { doActualSync, getSyncPlan, isPasswordOk } from "./sync";
 import { messyConfigToNormal, normalConfigToMessy } from "./configPersist";
+import { ObsConfigDirFileType, listFilesInObsFolder } from "./obsFolderLister";
 
 import * as origLog from "loglevel";
 import { DeletionOnRemote, MetadataOnRemote } from "./metadataOnRemote";
@@ -56,6 +57,8 @@ const DEFAULT_SETTINGS: RemotelySavePluginSettings = {
   initRunAfterMilliseconds: -1,
   agreeToUploadExtraMetadata: false,
   concurrency: 5,
+  syncConfigDir: false,
+  syncUnderscoreItems: false,
 };
 
 interface OAuth2Info {
@@ -190,6 +193,14 @@ export default class RemotelySavePlugin extends Plugin {
         this.db,
         this.settings.vaultRandomID
       );
+      let localConfigDirContents: ObsConfigDirFileType[] = undefined;
+      if (this.settings.syncConfigDir) {
+        localConfigDirContents = await listFilesInObsFolder(
+          this.app.vault.configDir,
+          this.app.vault,
+          this.manifest.id
+        );
+      }
       // log.info(local);
       // log.info(localHistory);
 
@@ -198,10 +209,14 @@ export default class RemotelySavePlugin extends Plugin {
       const { plan, sortedKeys, deletions } = await getSyncPlan(
         remoteStates,
         local,
+        localConfigDirContents,
         origMetadataOnRemote.deletions,
         localHistory,
         client.serviceType,
         this.app.vault,
+        this.settings.syncConfigDir,
+        this.app.vault.configDir,
+        this.settings.syncUnderscoreItems,
         this.settings.password
       );
       log.info(plan.mixedStates); // for debugging

src/misc.ts

@@ -10,10 +10,18 @@ const log = origLog.getLogger("rs-default");
 /**
  * If any part of the file starts with '.' or '_' then it's a hidden file.
  * @param item
- * @param loose
+ * @param dot
+ * @param underscore
  * @returns
  */
-export const isHiddenPath = (item: string, loose: boolean = true) => {
+export const isHiddenPath = (
+  item: string,
+  dot: boolean = true,
+  underscore: boolean = true
+) => {
+  if (!(dot || underscore)) {
+    throw Error("parameter error for isHiddenPath");
+  }
   const k = path.posix.normalize(item); // TODO: only unix path now
   const k2 = k.split("/"); // TODO: only unix path now
   // log.info(k2)
@@ -21,10 +29,10 @@ export const isHiddenPath = (item: string, loose: boolean = true) => {
     if (singlePart === "." || singlePart === ".." || singlePart === "") {
       continue;
     }
-    if (singlePart[0] === ".") {
+    if (dot && singlePart[0] === ".") {
       return true;
     }
-    if (loose && singlePart[0] === "_") {
+    if (underscore && singlePart[0] === "_") {
       return true;
     }
   }

src/obsFolderLister.ts

@@ -0,0 +1,127 @@
+import { Vault, Stat, ListedFiles } from "obsidian";
+import { Queue } from "@fyears/tsqueue";
+import chunk from "lodash/chunk";
+import flatten from "lodash/flatten";
+
+import * as origLog from "loglevel";
+const log = origLog.getLogger("rs-default");
+
+export interface ObsConfigDirFileType {
+  key: string;
+  ctime: number;
+  mtime: number;
+  size: number;
+  type: "folder" | "file";
+}
+
+const isFolderToSkip = (x: string) => {
+  let specialFolders = [".git", ".svn", "node_modules"];
+  for (const iterator of specialFolders) {
+    if (
+      x === iterator ||
+      x === `${iterator}/` ||
+      x.endsWith(`/${iterator}`) ||
+      x.endsWith(`/${iterator}/`)
+    ) {
+      return true;
+    }
+  }
+  return false;
+};
+
+const isPluginDirItself = (x: string, pluginId: string) => {
+  return (
+    x === pluginId ||
+    x === `${pluginId}/` ||
+    x.endsWith(`/${pluginId}`) ||
+    x.endsWith(`/${pluginId}/`)
+  );
+};
+
+const isLikelyPluginSubFiles = (x: string) => {
+  const reqFiles = [
+    "data.json",
+    "main.js",
+    "manifest.json",
+    ".gitignore",
+    "styles.css",
+  ];
+  for (const iterator of reqFiles) {
+    if (x === iterator || x.endsWith(`/${iterator}`)) {
+      return true;
+    }
+  }
+  return false;
+};
+
+export const isInsideObsFolder = (x: string, configDir: string) => {
+  if (!configDir.startsWith(".")) {
+    throw Error(`configDir should starts with . but we get ${configDir}`);
+  }
+  return x === configDir || x.startsWith(`${configDir}/`);
+};
+
+export const listFilesInObsFolder = async (
+  configDir: string,
+  vault: Vault,
+  pluginId: string
+) => {
+  const q = new Queue([configDir]);
+  const CHUNK_SIZE = 10;
+  const contents: ObsConfigDirFileType[] = [];
+  while (q.length > 0) {
+    const itemsToFetch = [];
+    while (q.length > 0) {
+      itemsToFetch.push(q.pop());
+    }
+
+    const itemsToFetchChunks = chunk(itemsToFetch, CHUNK_SIZE);
+    for (const singleChunk of itemsToFetchChunks) {
+      const r = singleChunk.map(async (x) => {
+        const statRes = await vault.adapter.stat(x);
+        const isFolder = statRes.type === "folder";
+        let children: ListedFiles = undefined;
+        if (isFolder) {
+          children = await vault.adapter.list(x);
+        }
+
+        return {
+          itself: {
+            key: isFolder ? `${x}/` : x,
+            ...statRes,
+          } as ObsConfigDirFileType,
+          children: children,
+        };
+      });
+      const r2 = flatten(await Promise.all(r));
+
+      for (const iter of r2) {
+        contents.push(iter.itself);
+        const isInsideSelfPlugin = isPluginDirItself(iter.itself.key, pluginId);
+        if (iter.children !== undefined) {
+          for (const iter2 of iter.children.folders) {
+            if (isFolderToSkip(iter2)) {
+              continue;
+            }
+            if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) {
+              // special treatment for remotely-save folder
+              continue;
+            }
+            q.push(iter2);
+          }
+          for (const iter2 of iter.children.files) {
+            if (isFolderToSkip(iter2)) {
+              continue;
+            }
+            if (isInsideSelfPlugin && !isLikelyPluginSubFiles(iter2)) {
+              // special treatment for remotely-save folder
+              continue;
+            }
+            q.push(iter2);
+          }
+        }
+      }
+    }
+  }
+  return contents;
+};

src/remoteForWebdav.ts

@@ -400,7 +400,10 @@ export const listFromRemote = async (
           return client.client.getDirectoryContents(x, {
             deep: false,
             details: false /* no need for verbose details here */,
-            glob: "/**" /* avoid dot files by using glob */,
+            // TODO: to support .obsidian,
+            // we need to load all files including dot,
+            // anyway to reduce the resources?
+            // glob: "/**" /* avoid dot files by using glob */,
           }) as Promise<FileStat[]>;
         });
         const r2 = flatten(await Promise.all(r));
@@ -421,7 +424,10 @@ export const listFromRemote = async (
       {
         deep: true,
         details: false /* no need for verbose details here */,
-        glob: "/**" /* avoid dot files by using glob */,
+        // TODO: to support .obsidian,
+        // we need to load all files including dot,
+        // anyway to reduce the resources?
+        // glob: "/**" /* avoid dot files by using glob */,
       }
     )) as FileStat[];
   }

src/settings.ts

@@ -384,6 +384,59 @@ export class OnedriveRevokeAuthModal extends Modal {
   }
 }
 
+class SyncConfigDirModal extends Modal {
+  plugin: RemotelySavePlugin;
+  saveDropdownFunc: () => void;
+  constructor(
+    app: App,
+    plugin: RemotelySavePlugin,
+    saveDropdownFunc: () => void
+  ) {
+    super(app);
+    this.plugin = plugin;
+    this.saveDropdownFunc = saveDropdownFunc;
+  }
+
+  async onOpen() {
+    let { contentEl } = this;
+
+    const texts = [
+      "Attention 1/3: This only syncs (copies) the whole Obsidian config dir, not other . folders or files. It also doesn't understand the inner structure of the config dir.",
+      "Attention 2/3: After the config dir is synced, plugins settings might be corrupted, and Obsidian might need to be restarted to load the new settings.",
+      "Attention 3/3: The deletion (uninstallation) operations of or inside Obsidian config dir cannot be tracked. So if you want to uninstall a plugin, you need to manually uninstall it on all device, before next sync.",
+      "If you are agreed to take your own risk, please click the following second confirm button.",
+    ];
+    for (const t of texts) {
+      contentEl.createEl("p", {
+        text: t,
+      });
+    }
+
+    new Setting(contentEl)
+      .addButton((button) => {
+        button.setButtonText("The Second Confirm To Enable.");
+        button.onClick(async () => {
+          this.plugin.settings.syncConfigDir = true;
+          await this.plugin.saveSettings();
+          this.saveDropdownFunc();
+          new Notice("You've enabled syncing config folder!");
+          this.close();
+        });
+      })
+      .addButton((button) => {
+        button.setButtonText("Go Back");
+        button.onClick(() => {
+          this.close();
+        });
+      });
+  }
+
+  onClose() {
+    let { contentEl } = this;
+    contentEl.empty();
+  }
+}
+
 class ExportSettingsQrCodeModal extends Modal {
   plugin: RemotelySavePlugin;
   constructor(app: App, plugin: RemotelySavePlugin) {
@@ -552,30 +605,6 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
-    const concurrencyDiv = generalDiv.createEl("div");
-    new Setting(concurrencyDiv)
-      .setName("Concurrency")
-      .setDesc(
-        "How many files do you want to download or upload in parallel at most? By default it's set to 5. If you meet any problems such as rate limit, you can reduce the concurrency to a lower value."
-      )
-      .addDropdown((dropdown) => {
-        dropdown.addOption("1", "1");
-        dropdown.addOption("2", "2");
-        dropdown.addOption("3", "3");
-        dropdown.addOption("5", "5 (default)");
-        dropdown.addOption("10", "10");
-        dropdown.addOption("15", "15");
-        dropdown.addOption("20", "20");
-
-        dropdown
-          .setValue(`${this.plugin.settings.concurrency}`)
-          .onChange(async (val) => {
-            const realVal = parseInt(val);
-            this.plugin.settings.concurrency = realVal;
-            await this.plugin.saveSettings();
-          });
-      });
-
     //////////////////////////////////////////////////
     // below for general chooser (part 1/2)
     //////////////////////////////////////////////////
@@ -1182,6 +1211,92 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           });
       });
 
+    //////////////////////////////////////////////////
+    // below for advanced settings
+    //////////////////////////////////////////////////
+    const advDiv = containerEl.createEl("div");
+    advDiv.createEl("h2", {
+      text: "Advanced Settings",
+    });
+
+    const concurrencyDiv = advDiv.createEl("div");
+    new Setting(concurrencyDiv)
+      .setName("Concurrency")
+      .setDesc(
+        "How many files do you want to download or upload in parallel at most? By default it's set to 5. If you meet any problems such as rate limit, you can reduce the concurrency to a lower value."
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("1", "1");
+        dropdown.addOption("2", "2");
+        dropdown.addOption("3", "3");
+        dropdown.addOption("5", "5 (default)");
+        dropdown.addOption("10", "10");
+        dropdown.addOption("15", "15");
+        dropdown.addOption("20", "20");
+
+        dropdown
+          .setValue(`${this.plugin.settings.concurrency}`)
+          .onChange(async (val) => {
+            const realVal = parseInt(val);
+            this.plugin.settings.concurrency = realVal;
+            await this.plugin.saveSettings();
+          });
+      });
+
+    const syncUnderscoreItemsDiv = advDiv.createEl("div");
+    new Setting(syncUnderscoreItemsDiv)
+      .setName("sync _ files or folders")
+      .setDesc(`Sync files or folders startting with _ ("underscore") or not.`)
+      .addDropdown((dropdown) => {
+        dropdown.addOption("disable", "disable");
+        dropdown.addOption("enable", "enable");
+        dropdown
+          .setValue(
+            `${this.plugin.settings.syncUnderscoreItems ? "enable" : "disable"}`
+          )
+          .onChange(async (val) => {
+            this.plugin.settings.syncUnderscoreItems = val === "enable";
+            await this.plugin.saveSettings();
+          });
+      });
+
+    const syncConfigDirDiv = advDiv.createEl("div");
+    new Setting(syncConfigDirDiv)
+      .setName("sync config dir (experimental)")
+      .setDesc(
+        `Sync config dir ${this.app.vault.configDir} or not. Please be aware that this may impact all your plugins' or Obsidian's settings, and may require you restart Obsidian after sync. Enable this at your own risk.`
+      )
+      .addDropdown((dropdown) => {
+        dropdown.addOption("disable", "disable");
+        dropdown.addOption("enable", "enable");
+
+        const bridge = {
+          secondConfirm: false,
+        };
+        dropdown
+          .setValue(
+            `${this.plugin.settings.syncConfigDir ? "enable" : "disable"}`
+          )
+          .onChange(async (val) => {
+            if (val === "enable" && !bridge.secondConfirm) {
+              dropdown.setValue("disable");
+              const modal = new SyncConfigDirModal(
+                this.app,
+                this.plugin,
+                () => {
+                  bridge.secondConfirm = true;
+                  dropdown.setValue("enable");
+                }
+              );
+              modal.open();
+            } else {
+              bridge.secondConfirm = false;
+              this.plugin.settings.syncConfigDir = false;
+              await this.plugin.saveSettings();
+            }
+          });
+      });
+
     //////////////////////////////////////////////////
     // below for import and export functions
     //////////////////////////////////////////////////

src/sync.ts

@@ -46,6 +46,7 @@ import {
 
 import * as origLog from "loglevel";
 import { padEnd } from "lodash";
+import { isInsideObsFolder, ObsConfigDirFileType } from "./obsFolderLister";
 const log = origLog.getLogger("rs-default");
 
 export type SyncStatusType =
@@ -272,18 +273,39 @@ export const fetchMetadataFile = async (
   return metadata;
 };
 
+const isSkipItem = (
+  key: string,
+  syncConfigDir: boolean,
+  syncUnderscoreItems: boolean,
+  configDir: string
+) => {
+  if (syncConfigDir && isInsideObsFolder(key, configDir)) {
+    return false;
+  }
+  return (
+    isHiddenPath(key, true, false) ||
+    (!syncUnderscoreItems && isHiddenPath(key, false, true)) ||
+    key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE ||
+    key === DEFAULT_FILE_NAME_FOR_METADATAONREMOTE2
+  );
+};
+
 const ensembleMixedStates = async (
   remoteStates: FileOrFolderMixedState[],
   local: TAbstractFile[],
+  localConfigDirContents: ObsConfigDirFileType[] | undefined,
   remoteDeleteHistory: DeletionOnRemote[],
-  localDeleteHistory: FileFolderHistoryRecord[]
+  localDeleteHistory: FileFolderHistoryRecord[],
+  syncConfigDir: boolean,
+  configDir: string,
+  syncUnderscoreItems: boolean
 ) => {
   const results = {} as Record<string, FileOrFolderMixedState>;
 
   for (const r of remoteStates) {
     const key = r.key;
 
-    if (isHiddenPath(key)) {
+    if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
       continue;
     }
     results[key] = r;
@@ -316,9 +338,10 @@ const ensembleMixedStates = async (
       throw Error(`unexpected ${entry}`);
     }
 
-    if (isHiddenPath(key)) {
+    if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
       continue;
     }
+
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].existLocal = r.existLocal;
@@ -330,6 +353,28 @@ const ensembleMixedStates = async (
     }
   }
 
+  if (syncConfigDir && localConfigDirContents !== undefined) {
+    for (const entry of localConfigDirContents) {
+      const key = entry.key;
+      const r: FileOrFolderMixedState = {
+        key: key,
+        existLocal: true,
+        mtimeLocal: Math.max(entry.mtime, entry.ctime),
+        sizeLocal: entry.size,
+      };
+
+      if (results.hasOwnProperty(key)) {
+        results[key].key = r.key;
+        results[key].existLocal = r.existLocal;
+        results[key].mtimeLocal = r.mtimeLocal;
+        results[key].sizeLocal = r.sizeLocal;
+      } else {
+        results[key] = r;
+        results[key].existRemote = false;
+      }
+    }
+  }
+
   for (const entry of remoteDeleteHistory) {
     const key = entry.key;
     const r = {
@@ -337,6 +382,10 @@ const ensembleMixedStates = async (
       deltimeRemote: entry.actionWhen,
     } as FileOrFolderMixedState;
 
+    if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
+      continue;
+    }
+
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].deltimeRemote = r.deltimeRemote;
@@ -365,9 +414,10 @@ const ensembleMixedStates = async (
       deltimeLocal: entry.actionWhen,
     } as FileOrFolderMixedState;
 
-    if (isHiddenPath(key)) {
+    if (isSkipItem(key, syncConfigDir, syncUnderscoreItems, configDir)) {
       continue;
     }
+
     if (results.hasOwnProperty(key)) {
       results[key].key = r.key;
       results[key].deltimeLocal = r.deltimeLocal;
@@ -618,17 +668,25 @@ const DELETION_DECISIONS: Set<DecisionType> = new Set([
 export const getSyncPlan = async (
   remoteStates: FileOrFolderMixedState[],
   local: TAbstractFile[],
+  localConfigDirContents: ObsConfigDirFileType[] | undefined,
   remoteDeleteHistory: DeletionOnRemote[],
   localDeleteHistory: FileFolderHistoryRecord[],
   remoteType: SUPPORTED_SERVICES_TYPE,
   vault: Vault,
+  syncConfigDir: boolean,
+  configDir: string,
+  syncUnderscoreItems: boolean,
   password: string = ""
 ) => {
   const mixedStates = await ensembleMixedStates(
     remoteStates,
     local,
+    localConfigDirContents,
     remoteDeleteHistory,
-    localDeleteHistory
+    localDeleteHistory,
+    syncConfigDir,
+    configDir,
+    syncUnderscoreItems
   );
 
   const sortedKeys = Object.keys(mixedStates).sort(

tests/misc.test.ts

@@ -21,7 +21,7 @@ describe("Misc: hidden file", () => {
 
     item = "_hidden_loose";
     expect(misc.isHiddenPath(item)).to.be.true;
-    expect(misc.isHiddenPath(item, false)).to.be.false;
+    expect(misc.isHiddenPath(item, true, false)).to.be.false;
 
     item = "/sdd/_hidden_loose";
     expect(misc.isHiddenPath(item)).to.be.true;
@@ -30,10 +30,21 @@ describe("Misc: hidden file", () => {
     expect(misc.isHiddenPath(item)).to.be.true;
 
     item = "what/../_hidden_loose/what/what/what";
-    expect(misc.isHiddenPath(item, false)).to.be.false;
+    expect(misc.isHiddenPath(item, true, false)).to.be.false;
 
     item = "what/../_hidden_loose/../.hidden/what/what/what";
-    expect(misc.isHiddenPath(item, false)).to.be.true;
+    expect(misc.isHiddenPath(item, true, false)).to.be.true;
+
+    item = "what/../_hidden_loose/../.hidden/what/what/what";
+    expect(misc.isHiddenPath(item, false, true)).to.be.false;
+
+    item = "what/_hidden_loose/what/what/what";
+    expect(misc.isHiddenPath(item, false, true)).to.be.true;
+    expect(misc.isHiddenPath(item, true, false)).to.be.false;
+
+    item = "what/.hidden/what/what/what";
+    expect(misc.isHiddenPath(item, false, true)).to.be.false;
+    expect(misc.isHiddenPath(item, true, false)).to.be.true;
   });
 });