Improve schema checking path for deploy2 (#29394)

GitOrigin-RevId: d244fa200dc2687b066abc5439e222679192589a

src/cli/lib/codegen.ts

@@ -142,7 +142,12 @@ export async function doInitialComponentCodegen(
   ctx: Context,
   tmpDir: TempDir,
   componentDirectory: ComponentDirectory,
-  opts?: { dryRun?: boolean; generateCommonJSApi?: boolean; debug?: boolean },
+  opts?: {
+    dryRun?: boolean;
+    generateCommonJSApi?: boolean;
+    debug?: boolean;
+    verbose?: boolean;
+  },
 ) {
   const { projectConfig } = await readProjectConfig(ctx);
 
@@ -156,7 +161,7 @@ export async function doInitialComponentCodegen(
   const isPublishedPackage =
     componentDirectory.definitionPath.endsWith(".js") &&
     !componentDirectory.isRoot;
-  if (isPublishedPackage) {
+  if (isPublishedPackage && opts?.verbose) {
     logMessage(
       ctx,
       `skipping initial codegen for installed package ${componentDirectory.path}`,

src/cli/lib/components.ts

@@ -129,7 +129,7 @@ async function startComponentsPushAndCodegen(
   }
   const rootComponent = isComponent.component;
 
-  changeSpinner(ctx, "Traversing component definitions...");
+  changeSpinner(ctx, "Finding component definitions...");
   // Create a list of relevant component directories. These are just for knowing
   // while directories to bundle in bundleDefinitions and bundleImplementations.
   // This produces a bundle in memory as a side effect but it's thrown away.
@@ -238,6 +238,7 @@ async function startComponentsPushAndCodegen(
     return null;
   }
 
+  changeSpinner(ctx, "Uploading functions to Convex...");
   const startPushResponse = await startPush(
     ctx,
     options.url,
@@ -248,7 +249,7 @@ async function startComponentsPushAndCodegen(
   verbose && console.log("startPush:");
   verbose && console.dir(startPushResponse, { depth: null });
 
-  changeSpinner(ctx, "Finalizing code generation...");
+  changeSpinner(ctx, "Generating TypeScript bindings...");
   await withTmpDir(async (tmpDir) => {
     await doFinalComponentCodegen(
       ctx,
@@ -306,7 +307,6 @@ export async function runComponentsPush(
     return;
   }
 
-  changeSpinner(ctx, "Waiting for schema...");
   await waitForSchema(ctx, options.adminKey, options.url, startPushResponse);
 
   const finishPushResponse = await finishPush(

src/cli/lib/deploy2.ts

@@ -1,6 +1,13 @@
-import { changeSpinner, Context, logFailure } from "../../bundler/context.js";
+import {
+  changeSpinner,
+  Context,
+  logError,
+  logFailure,
+} from "../../bundler/context.js";
 import { deploymentFetch, logAndHandleFetchError } from "./utils/utils.js";
 import {
+  schemaStatus,
+  SchemaStatus,
   StartPushRequest,
   startPushResponse,
   StartPushResponse,
@@ -9,6 +16,7 @@ import {
   AppDefinitionConfig,
   ComponentDefinitionConfig,
 } from "./deployApi/definitionConfig.js";
+import chalk from "chalk";
 
 /** Push configuration2 to the given remote origin. */
 export async function startPush(
@@ -42,27 +50,89 @@ export async function startPush(
   }
 }
 
+// Long poll every 10s for progress on schema validation.
+const SCHEMA_TIMEOUT_MS = 10_000;
+
 export async function waitForSchema(
   ctx: Context,
   adminKey: string,
   url: string,
   startPush: StartPushResponse,
 ) {
   const fetch = deploymentFetch(url, adminKey);
-  try {
-    const response = await fetch("/api/deploy2/wait_for_schema", {
-      body: JSON.stringify({
-        adminKey,
-        schemaChange: (startPush as any).schemaChange,
-        dryRun: false,
-      }),
-      method: "POST",
-    });
-    return await response.json();
-  } catch (error: unknown) {
-    // TODO incorporate AuthConfigMissingEnvironmentVariable logic
-    logFailure(ctx, "Error: Unable to wait for schema from " + url);
-    return await logAndHandleFetchError(ctx, error);
+
+  changeSpinner(
+    ctx,
+    "Backfilling indexes and checking that documents match your schema...",
+  );
+
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    let currentStatus: SchemaStatus;
+    try {
+      const response = await fetch("/api/deploy2/wait_for_schema", {
+        body: JSON.stringify({
+          adminKey,
+          schemaChange: startPush.schemaChange,
+          timeoutMs: SCHEMA_TIMEOUT_MS,
+        }),
+        method: "POST",
+      });
+      currentStatus = schemaStatus.parse(await response.json());
+    } catch (error: unknown) {
+      logFailure(ctx, "Error: Unable to wait for schema from " + url);
+      return await logAndHandleFetchError(ctx, error);
+    }
+    switch (currentStatus.type) {
+      case "inProgress": {
+        let schemaDone = true;
+        let indexesComplete = 0;
+        let indexesTotal = 0;
+        for (const componentStatus of Object.values(currentStatus.components)) {
+          if (!componentStatus.schemaValidationComplete) {
+            schemaDone = false;
+          }
+          indexesComplete += componentStatus.indexesComplete;
+          indexesTotal += componentStatus.indexesTotal;
+        }
+        const indexesDone = indexesComplete === indexesTotal;
+        let msg: string;
+        if (!indexesDone && !schemaDone) {
+          msg = `Backfilling indexes (${indexesComplete}/${indexesTotal} ready) and checking that documents match your schema...`;
+        } else if (!indexesDone) {
+          msg = `Backfilling indexes (${indexesComplete}/${indexesTotal} ready)...`;
+        } else {
+          msg = "Checking that documents match your schema...";
+        }
+        changeSpinner(ctx, msg);
+        break;
+      }
+      case "failed": {
+        // Schema validation failed. This could be either because the data
+        // is bad or the schema is wrong. Classify this as a filesystem error
+        // because adjusting `schema.ts` is the most normal next step.
+        logFailure(ctx, "Schema validation failed");
+        logError(ctx, chalk.red(`${currentStatus.error}`));
+        return await ctx.crash({
+          exitCode: 1,
+          errorType: {
+            "invalid filesystem or db data": currentStatus.tableName ?? null,
+          },
+          printedMessage: null, // TODO - move logging into here
+        });
+      }
+      case "raceDetected": {
+        return await ctx.crash({
+          exitCode: 1,
+          errorType: "fatal",
+          printedMessage: `Schema was overwritten by another push.`,
+        });
+      }
+      case "complete": {
+        changeSpinner(ctx, "Schema validation complete.");
+        return;
+      }
+    }
   }
 }
 
@@ -72,8 +142,8 @@ export async function finishPush(
   url: string,
   startPush: StartPushResponse,
 ): Promise<void> {
-  const fetch = deploymentFetch(url, adminKey);
   changeSpinner(ctx, "Finalizing push...");
+  const fetch = deploymentFetch(url, adminKey);
   try {
     const response = await fetch("/api/deploy2/finish_push", {
       body: JSON.stringify({

src/cli/lib/deployApi/startPush.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { componentDefinitionPath } from "./paths.js";
+import { componentDefinitionPath, componentPath } from "./paths.js";
 import { nodeDependency, sourcePackage } from "./modules.js";
 import { checkedComponent } from "./checkedComponent.js";
 import { evaluatedComponentDefinition } from "./componentDefinition.js";
@@ -40,3 +40,30 @@ export const startPushResponse = z.object({
   schemaChange,
 });
 export type StartPushResponse = z.infer<typeof startPushResponse>;
+
+export const componentSchemaStatus = z.object({
+  schemaValidationComplete: z.boolean(),
+  indexesComplete: z.number(),
+  indexesTotal: z.number(),
+});
+export type ComponentSchemaStatus = z.infer<typeof componentSchemaStatus>;
+
+export const schemaStatus = z.union([
+  z.object({
+    type: z.literal("inProgress"),
+    components: z.record(componentPath, componentSchemaStatus),
+  }),
+  z.object({
+    type: z.literal("failed"),
+    error: z.string(),
+    componentPath,
+    tableName: z.nullable(z.string()),
+  }),
+  z.object({
+    type: z.literal("raceDetected"),
+  }),
+  z.object({
+    type: z.literal("complete"),
+  }),
+]);
+export type SchemaStatus = z.infer<typeof schemaStatus>;