Save text files directly to the DB for performance (goplus#617)

Updates goplus#369

spx-gui/src/apis/common/index.ts

@@ -17,10 +17,10 @@ export enum IsPublic {
   public = 1
 }
 
-/** Url with 'http://' or 'https://' scheme, used for web resources */
+/** Url with 'http:', 'https:', or 'data:' schemes, used for web resources that can be accessed directly via `fetch()` */
 export type WebUrl = string
 
-/** Url for universal resources, which could be either a WebUrl or a Url with a custom scheme like 'kodo://' */
+/** Url for universal resources, which could be either a WebUrl or a Url with a custom scheme like 'kodo:' */
 export type UniversalUrl = string
 
 /** Map from UniversalUrl to WebUrl */

spx-gui/src/components/project/ProjectCreateModal.vue

@@ -52,7 +52,7 @@ import { ApiException, ApiExceptionCode } from '@/apis/common/exception'
 import { Sprite } from '@/models/sprite'
 import { Costume } from '@/models/costume'
 import { File } from '@/models/common/file'
-import { uploadFiles } from '@/models/common/cloud'
+import { saveFiles } from '@/models/common/cloud'
 import { filename } from '@/utils/path'
 import defaultSpritePng from '@/assets/default-sprite.png'
 import defaultBackdropImg from '@/assets/default-backdrop.png'
@@ -106,7 +106,7 @@ const handleSubmit = useMessageHandle(
     await sprite.autoFit()
     // upload project content & call API addProject, TODO: maybe this should be extracted to `@/models`?
     const files = project.export()[1]
-    const { fileCollection } = await uploadFiles(files)
+    const { fileCollection } = await saveFiles(files)
     const projectData = await addProject({
       name: form.value.name,
       isPublic: IsPublic.personal,

spx-gui/src/models/common/asset.ts

@@ -3,7 +3,7 @@ import { fromConfig, toConfig } from './file'
 import { Sound } from '../sound'
 import { Sprite } from '../sprite'
 import { Backdrop, type BackdropInits } from '../backdrop'
-import { getFiles, uploadFiles } from './cloud'
+import { getFiles, saveFiles } from './cloud'
 
 export type PartialAssetData = Pick<AssetData, 'displayName' | 'assetType' | 'files' | 'filesHash'>
 
@@ -16,7 +16,7 @@ export type AssetModel<T extends AssetType = AssetType> = T extends AssetType.So
       : never
 
 export async function sprite2Asset(sprite: Sprite): Promise<PartialAssetData> {
-  const { fileCollection, fileCollectionHash } = await uploadFiles(sprite.export(false))
+  const { fileCollection, fileCollectionHash } = await saveFiles(sprite.export(false))
   return {
     displayName: sprite.name,
     assetType: AssetType.Sprite,
@@ -38,7 +38,7 @@ const virtualBackdropConfigFileName = 'assets/__backdrop__.json'
 export async function backdrop2Asset(backdrop: Backdrop): Promise<PartialAssetData> {
   const [config, files] = backdrop.export()
   files[virtualBackdropConfigFileName] = fromConfig(virtualBackdropConfigFileName, config)
-  const { fileCollection, fileCollectionHash } = await uploadFiles(files)
+  const { fileCollection, fileCollectionHash } = await saveFiles(files)
   return {
     displayName: backdrop.name,
     assetType: AssetType.Backdrop,
@@ -56,7 +56,7 @@ export async function asset2Backdrop(assetData: PartialAssetData) {
 }
 
 export async function sound2Asset(sound: Sound): Promise<PartialAssetData> {
-  const { fileCollection, fileCollectionHash } = await uploadFiles(sound.export())
+  const { fileCollection, fileCollectionHash } = await saveFiles(sound.export())
   return {
     displayName: sound.name,
     assetType: AssetType.Sound,

spx-gui/src/models/common/cloud.ts

@@ -6,11 +6,21 @@ import { IsPublic, addProject, getProject, updateProject } from '@/apis/project'
 import { getUpInfo as getRawUpInfo, makeObjectUrls, type UpInfo as RawUpInfo } from '@/apis/util'
 import { DefaultException } from '@/utils/exception'
 import type { Metadata } from '../project'
-import { File, toNativeFile, type Files } from './file'
+import { File, toNativeFile, toText, type Files } from './file'
 import { hashFileCollection } from './hash'
 
-// See https://github.com/goplus/builder/issues/411 for all the supported schemes, future plans, and discussions.
-const kodoScheme = 'kodo://'
+// Supported universal Url schemes for files
+const fileUniversalUrlSchemes = {
+  // for resources stored in third-party services
+  http: 'http:',
+  https: 'https:',
+
+  data: 'data:', // for inlineable data, usually plain text or json, e.g. data:text/plain,hello%20world
+  kodo: 'kodo:' // for objects stored in Qiniu Kodo, e.g. kodo://bucket/key
+} as const
+
+// File types that can be inlined in the Data Urls
+const inlineableFileTypes = ['text/plain', 'application/json']
 
 export async function load(owner: string, name: string) {
   const projectData = await getProject(owner, name)
@@ -21,7 +31,7 @@ export async function save(metadata: Metadata, files: Files) {
   const { owner, name, id } = metadata
   if (owner == null) throw new Error('owner expected')
   if (!name) throw new DefaultException({ en: 'project name not specified', zh: '未指定项目名' })
-  const { fileCollection } = await uploadFiles(files)
+  const { fileCollection } = await saveFiles(files)
   const isPublic = metadata.isPublic ?? IsPublic.personal
   const projectData = await (id != null
     ? updateProject(owner, name, { isPublic, files: fileCollection })
@@ -34,28 +44,34 @@ export async function parseProjectData({ files: fileCollection, ...metadata }: P
   return { metadata, files }
 }
 
-export async function uploadFiles(
+export async function saveFiles(
   files: Files
 ): Promise<{ fileCollection: FileCollection; fileCollectionHash: string }> {
   const fileCollection: FileCollection = {}
-  const entries = await Promise.all(
-    Object.keys(files).map(async (path) => [path, await uploadFile(files[path]!)] as const)
-  )
-  for (const [path, url] of entries) {
-    fileCollection[path] = url
+  for (const [path, file] of Object.entries(files)) {
+    if (!file) continue
+    if (inlineableFileTypes.includes(file.type)) {
+      // Little trick from [https://fetch.spec.whatwg.org/#data-urls]: `12. If mimeType starts with ';', then prepend 'text/plain' to mimeType.`
+      // Saves some bytes.
+      const mimeType = file.type === 'text/plain' ? ';' : file.type
+
+      const urlEncodedContent = encodeURIComponent(await toText(file))
+      fileCollection[path] = `${fileUniversalUrlSchemes.data}${mimeType},${urlEncodedContent}`
+    } else {
+      fileCollection[path] = await uploadFile(file)
+    }
   }
   const fileCollectionHash = await hashFileCollection(fileCollection)
   return { fileCollection, fileCollectionHash }
 }
 
 export async function getFiles(fileCollection: FileCollection): Promise<Files> {
-  let objectUrls: UniversalToWebUrlMap = {}
-  const objectUniversalUrls = Object.values(fileCollection).filter((url) =>
-    url.startsWith(kodoScheme)
+  const objectUniversalUrls = Object.values(fileCollection).filter(
+    (url) => new URL(url).protocol === fileUniversalUrlSchemes.kodo
   )
-  if (objectUniversalUrls.length) {
-    objectUrls = await makeObjectUrls(objectUniversalUrls)
-  }
+  const objectUrls: UniversalToWebUrlMap = objectUniversalUrls.length
+    ? await makeObjectUrls(objectUniversalUrls)
+    : {}
 
   const files: Files = {}
   Object.keys(fileCollection).forEach((path) => {
@@ -77,7 +93,7 @@ export async function getFiles(fileCollection: FileCollection): Promise<Files> {
 function setUniversalUrl(file: File, url: UniversalUrl) {
   file.meta.universalUrl = url
   // for binary files stored in kodo, use universalUrl as hash to skip hash-calculating
-  if (!['text/plain', 'application/json'].includes(file.type) && file.meta.hash == null) {
+  if (new URL(url).protocol === fileUniversalUrlSchemes.kodo && file.meta.hash == null) {
     file.meta.hash = url
   }
 }
@@ -130,7 +146,7 @@ async function uploadToKodo(file: File): Promise<UniversalUrl> {
       }
     })
   })
-  return kodoScheme + bucket + '/' + key
+  return `${fileUniversalUrlSchemes.kodo}//${bucket}/${key}`
 }
 
 type UpInfo = Omit<RawUpInfo, 'expires'> & {