allowing s3 synth folder

src/baseTypes.ts

@@ -29,6 +29,8 @@ export interface S3Config {
   useAccurateMTime?: boolean;
   reverseProxyNoSignUrl?: string;
 
+  generateFolderObject?: boolean;
+
   /**
    * @deprecated
    */

src/fsEncrypt.ts

@@ -211,6 +211,7 @@ export class FakeFsEncrypt extends FakeFs {
           sizeEnc: innerEntity.size!,
           sizeRaw: innerEntity.sizeRaw,
           hash: undefined,
+          synthesizedFolder: innerEntity.synthesizedFolder,
         });
 
         this.cacheMapOrigToEnc[key] = innerEntity.keyRaw;
@@ -243,6 +244,7 @@ export class FakeFsEncrypt extends FakeFs {
         sizeEnc: innerEntity.size!,
         sizeRaw: innerEntity.sizeRaw,
         hash: undefined,
+        synthesizedFolder: innerEntity.synthesizedFolder,
       };
     }
   }
@@ -287,6 +289,7 @@ export class FakeFsEncrypt extends FakeFs {
         sizeEnc: innerEntity.size!,
         sizeRaw: innerEntity.sizeRaw,
         hash: undefined,
+        synthesizedFolder: innerEntity.synthesizedFolder,
       };
     }
   }
@@ -336,6 +339,7 @@ export class FakeFsEncrypt extends FakeFs {
         sizeEnc: innerEntity.size!,
         sizeRaw: innerEntity.sizeRaw,
         hash: undefined,
+        synthesizedFolder: innerEntity.synthesizedFolder,
       };
     }
   }

src/fsS3.ts

@@ -26,7 +26,7 @@ import { Readable } from "stream";
 import * as path from "path";
 import AggregateError from "aggregate-error";
 import { DEFAULT_CONTENT_TYPE, S3Config, VALID_REQURL } from "./baseTypes";
-import { bufferToArrayBuffer } from "./misc";
+import { bufferToArrayBuffer, getFolderLevels } from "./misc";
 import PQueue from "p-queue";
 
 import { Entity } from "./baseTypes";
@@ -186,6 +186,7 @@ export const DEFAULT_S3_CONFIG: S3Config = {
   remotePrefix: "",
   useAccurateMTime: false, // it causes money, disable by default
   reverseProxyNoSignUrl: "",
+  generateFolderObject: false, // new version, by default not generate folders
 };
 
 /**
@@ -385,11 +386,13 @@ export class FakeFsS3 extends FakeFs {
   s3Config: S3Config;
   s3Client: S3Client;
   kind: "s3";
+  synthFoldersCache: Record<string, Entity>;
   constructor(s3Config: S3Config) {
     super();
     this.s3Config = s3Config;
     this.s3Client = getS3Client(s3Config);
     this.kind = "s3";
+    this.synthFoldersCache = {};
   }
 
   async walk(): Promise<Entity[]> {
@@ -484,17 +487,52 @@ export class FakeFsS3 extends FakeFs {
     // ensemble fake rsp
     // in the end, we need to transform the response list
     // back to the local contents-alike list
-    return contents.map((x) =>
+    const res: Entity[] = [];
-      fromS3ObjectToEntity(
+    const realEnrities = new Set<string>();
-        x,
+    for (const remoteObj of contents) {
+      const remoteEntity = fromS3ObjectToEntity(
+        remoteObj,
         this.s3Config.remotePrefix ?? "",
         mtimeRecords,
         ctimeRecords
-      )
+      );
-    );
+      realEnrities.add(remoteEntity.key!);
+      res.push(remoteEntity);
+
+      for (const f of getFolderLevels(remoteEntity.key!, true)) {
+        if (realEnrities.has(f)) {
+          delete this.synthFoldersCache[f];
+          continue;
+        }
+        if (
+          !this.synthFoldersCache.hasOwnProperty(f) ||
+          remoteEntity.mtimeSvr! >= this.synthFoldersCache[f].mtimeSvr!
+        ) {
+          this.synthFoldersCache[f] = {
+            key: f,
+            keyRaw: f,
+            size: 0,
+            sizeRaw: 0,
+            sizeEnc: 0,
+            mtimeSvr: remoteEntity.mtimeSvr,
+            mtimeSvrFmt: remoteEntity.mtimeSvrFmt,
+            mtimeCli: remoteEntity.mtimeCli,
+            mtimeCliFmt: remoteEntity.mtimeCliFmt,
+            synthesizedFolder: true,
+          };
+        }
+      }
+    }
+    for (const key of Object.keys(this.synthFoldersCache)) {
+      res.push(this.synthFoldersCache[key]);
+    }
+    return res;
   }
 
   async stat(key: string): Promise<Entity> {
+    if (this.synthFoldersCache.hasOwnProperty(key)) {
+      return this.synthFoldersCache[key];
+    }
     let keyFullPath = key;
     keyFullPath = getRemoteWithPrefixPath(
       keyFullPath,
@@ -529,6 +567,23 @@ export class FakeFsS3 extends FakeFs {
     if (!key.endsWith("/")) {
       throw new Error(`You should not call mkdir on ${key}!`);
     }
+
+    const generateFolderObject = this.s3Config.generateFolderObject ?? false;
+    if (!generateFolderObject) {
+      const synth = {
+        key: key,
+        keyRaw: key,
+        size: 0,
+        sizeRaw: 0,
+        sizeEnc: 0,
+        mtimeSvr: mtime,
+        mtimeCli: mtime,
+        synthesizedFolder: true,
+      };
+      this.synthFoldersCache[key] = synth;
+      return synth;
+    }
+
     const uploadFile = getRemoteWithPrefixPath(
       key,
       this.s3Config.remotePrefix ?? ""
@@ -670,6 +725,12 @@ export class FakeFsS3 extends FakeFs {
     if (key === "/") {
       return;
     }
+
+    if (this.synthFoldersCache.hasOwnProperty(key)) {
+      delete this.synthFoldersCache[key];
+      return;
+    }
+
     const remoteFileName = getRemoteWithPrefixPath(
       key,
       this.s3Config.remotePrefix ?? ""

src/langs/en.json

@@ -183,6 +183,10 @@
   "settings_s3_urlstyle_desc": "Whether to force path-style URLs for S3 objects (e.g., https://s3.amazonaws.com/*/ instead of https://*.s3.amazonaws.com/).",
   "settings_s3_reverse_proxy_no_sign_url": "S3 Reverse Proxy (No Sign) Url (experimental)",
   "settings_s3_reverse_proxy_no_sign_url_desc": "S3 reverse proxy url without signature. This is useful if you use a revers proxy but do not change the original credential signature. No http(s):// prefix. Leave it blank if you don't know what it is.",
+  "settings_s3_generatefolderobject": "Generate Folder Object Or Not",
+  "settings_s3_generatefolderobject_desc": "S3 doesn't have \"real\" folder. If you set \"Generate\" here (or use old version), the plugin will upload a zero-byte object endding with \"/\" to represent the folder. In the new version, the plugin skips generating folder object by default.",
+  "settings_s3_generatefolderobject_notgenerate": "Not generate (default)",
+  "settings_s3_generatefolderobject_generate": "Generate",
   "settings_s3_connect_succ": "Great! The bucket can be accessed.",
   "settings_s3_connect_fail": "The S3 bucket cannot be reached.",
   "settings_dropbox": "Remote For Dropbox",

src/langs/zh_cn.json

@@ -182,6 +182,10 @@
   "settings_s3_urlstyle_desc": "是否对 S3 对象强制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。",
   "settings_s3_reverse_proxy_no_sign_url": "S3 反向代理（不签名）地址（实验性质）",
   "settings_s3_reverse_proxy_no_sign_url_desc": "不会参与到签名的 S3 反向代理地址。如果您有一个反向代理，但是不想修改原始鉴权签名，这里就可以填写。没有 http(s):// 前缀。如果您不知道这是什么，留空即可。",
+  "settings_s3_generatefolderobject": "是否生成文件夹 Object",
+  "settings_s3_generatefolderobject_desc": "S3 不存在“真正”的文件夹。如果您设置了“生成”（或用了旧版本），那么插件会上传 0 字节的以“/”结尾的 Object 来代表文件夹。新版本插件会默认跳过生成这种文件夹 Object。",
+  "settings_s3_generatefolderobject_notgenerate": "不生成（默认）",
+  "settings_s3_generatefolderobject_generate": "生成",
   "settings_s3_connect_succ": "很好！可以访问到对应存储桶。",
   "settings_s3_connect_fail": "无法访问到对应存储桶。",
   "settings_dropbox": "Dropbox 设置",

src/langs/zh_tw.json

@@ -181,6 +181,10 @@
   "settings_s3_urlstyle_desc": "是否對 S3 物件強制使用 path style URL（例如使用 https://s3.amazonaws.com/*/ 而不是 https://*.s3.amazonaws.com/）。",
   "settings_s3_reverse_proxy_no_sign_url": "S3 反向代理（不簽名）地址（實驗性質）",
   "settings_s3_reverse_proxy_no_sign_url_desc": "不會參與到簽名的 S3 反向代理地址。如果您有一個反向代理，但是不想修改原始鑑權簽名，這裡就可以填寫。沒有 http(s):// 字首。如果您不知道這是什麼，留空即可。",
+  "settings_s3_generatefolderobject": "是否生成文件夾 Object",
+  "settings_s3_generatefolderobject_desc": "S3 不存在“真正”的文件夾。如果您設置了“生成”（或用了舊版本），那麼插件會上傳 0 字節的以“/”結尾的 Object 來代表文件夾。新版本插件會默認跳過生成這種文件夾 Object。",
+  "settings_s3_generatefolderobject_notgenerate": "不生成（默認）",
+  "settings_s3_generatefolderobject_generate": "生成",
   "settings_s3_connect_succ": "很好！可以訪問到對應儲存桶。",
   "settings_s3_connect_fail": "無法訪問到對應儲存桶。",
   "settings_dropbox": "Dropbox 設定",

src/main.ts

@@ -868,6 +868,9 @@ export default class RemotelySavePlugin extends Plugin {
       // it causes money, so disable it by default
       this.settings.s3.useAccurateMTime = false;
     }
+    if (this.settings.s3.generateFolderObject === undefined) {
+      this.settings.s3.generateFolderObject = false;
+    }
     if (this.settings.ignorePaths === undefined) {
       this.settings.ignorePaths = [];
     }

src/settings.ts

@@ -1067,6 +1067,34 @@ export class RemotelySaveSettingTab extends PluginSettingTab {
           })
       );
 
+    new Setting(s3Div)
+      .setName(t("settings_s3_generatefolderobject"))
+      .setDesc(t("settings_s3_generatefolderobject_desc"))
+      .addDropdown((dropdown) => {
+        dropdown
+          .addOption(
+            "notgenerate",
+            t("settings_s3_generatefolderobject_notgenerate")
+          )
+          .addOption(
+            "generate",
+            t("settings_s3_generatefolderobject_generate")
+          );
+
+        dropdown
+          .setValue(
+            `${this.plugin.settings.s3.generateFolderObject ? "generate" : "notgenerate"}`
+          )
+          .onChange(async (val) => {
+            if (val === "generate") {
+              this.plugin.settings.s3.generateFolderObject = true;
+            } else {
+              this.plugin.settings.s3.generateFolderObject = false;
+            }
+            await this.plugin.saveSettings();
+          });
+      });
+
     new Setting(s3Div)
       .setName(t("settings_checkonnectivity"))
       .setDesc(t("settings_checkonnectivity_desc"))

src/sync.ts

@@ -20,7 +20,6 @@ import {
 } from "./localdb";
 import {
   atWhichLevel,
-  getFolderLevels,
   getParentFolder,
   isHiddenPath,
   isSpecialFolderNameToSkip,
@@ -161,10 +160,7 @@ const ensembleMixedEnties = async (
 
   const finalMappings: SyncPlanType = {};
 
-  const synthFolders: Record<string, Entity> = {};
-
   // remote has to be first
-  // we also have to synthesize folders here
   for (const remote of remoteEntityList) {
     const remoteCopied = ensureMTimeOfRemoteEntityValid(
       copyEntityAndFixTimeFormat(remote, serviceType)
@@ -187,48 +183,10 @@ const ensembleMixedEnties = async (
       key: key,
       remote: remoteCopied,
     };
-
-    for (const f of getFolderLevels(key, true)) {
-      if (finalMappings.hasOwnProperty(f)) {
-        delete synthFolders[f];
-        continue;
-      }
-      if (
-        !synthFolders.hasOwnProperty(f) ||
-        remoteCopied.mtimeSvr! >= synthFolders[f].mtimeSvr!
-      ) {
-        synthFolders[f] = {
-          key: f,
-          keyRaw: `<synth: ${f}>`,
-          keyEnc: `<enc synth: ${f}>`,
-          size: 0,
-          sizeRaw: 0,
-          sizeEnc: 0,
-          mtimeSvr: remoteCopied.mtimeSvr,
-          mtimeSvrFmt: remoteCopied.mtimeSvrFmt,
-          mtimeCli: remoteCopied.mtimeCli,
-          mtimeCliFmt: remoteCopied.mtimeCliFmt,
-          synthesizedFolder: true,
-        };
-      }
-    }
   }
 
   profiler.insert("ensembleMixedEnties: finish remote");
 
-  console.debug(`synthFolders:`);
-  console.debug(synthFolders);
-
-  // special: add synth folders
-  for (const key of Object.keys(synthFolders)) {
-    finalMappings[key] = {
-      key: key,
-      remote: synthFolders[key],
-    };
-  }
-
-  profiler.insert("ensembleMixedEnties: finish synth");
-
   if (Object.keys(finalMappings).length === 0 || localEntityList.length === 0) {
     // Special checking:
     // if one side is totally empty,