refactor(deploy): simplify deploy polling, retries, streams

eslint-suppressions.json

@@ -866,7 +866,7 @@
   },
   "src/utils/deploy/util.ts": {
     "@typescript-eslint/prefer-nullish-coalescing": {
-      "count": 2
+      "count": 1
     },
     "@typescript-eslint/restrict-template-expressions": {
       "count": 1

src/utils/deploy/drop-api.ts

@@ -121,42 +121,35 @@ export const waitForDropDeploy = async (
   siteId: string,
   deployId: string,
   timeout: number = DEFAULT_DEPLOY_TIMEOUT,
-): Promise<DropSiteDeploy> => {
-  let deploy: DropSiteDeploy | undefined
-
-  const checkDeploy = async (): Promise<boolean> => {
-    const response = await fetch(`${apiBase}/sites/${siteId}/deploys/${deployId}`, {
-      headers: makeHeaders(userAgent),
-    })
+): Promise<DropSiteDeploy> =>
+  await pWaitFor(
+    async () => {
+      const response = await fetch(`${apiBase}/sites/${siteId}/deploys/${deployId}`, {
+        headers: makeHeaders(userAgent),
+      })
+
+      if (!response.ok) {
+        return false
+      }
 
-    if (!response.ok) {
+      const data = (await response.json()) as DropSiteDeploy
+      if (data.state === 'ready') {
+        return pWaitFor.resolveWith(data)
+      }
+      if (data.state === 'error') {
+        // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- an empty error message falls back to the generic one
+        throw new Error(data.error_message || `Deploy ${deployId} had an error`)
+      }
       return false
-    }
-
-    const data = (await response.json()) as DropSiteDeploy
-    if (data.state === 'ready') {
-      deploy = data
-      return true
-    }
-    if (data.state === 'error') {
-      // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- an empty error message falls back to the generic one
-      throw new Error(data.error_message || `Deploy ${deployId} had an error`)
-    }
-    return false
-  }
-
-  await pWaitFor(checkDeploy, {
-    interval: DEPLOY_POLL,
-    timeout: {
-      milliseconds: timeout,
-      message: 'Timeout while waiting for deploy',
     },
-  })
-
-  // deploy is guaranteed to be set when pWaitFor resolves
-  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-  return deploy!
-}
+    {
+      interval: DEPLOY_POLL,
+      timeout: {
+        milliseconds: timeout,
+        message: 'Timeout while waiting for deploy',
+      },
+    },
+  )
 
 // TODO: Migrate to @netlify/api when Drop endpoints are in the OpenAPI spec.
 export const claimDropSite = async (
@@ -218,25 +211,20 @@ export const uploadDropFiles = async (
       phase: 'progress',
     })
 
-    let lastError: DropApiError | undefined
     for (let attempt = 0; attempt <= maxRetry; attempt++) {
       try {
         const body = fs.createReadStream(fileObj.filepath)
         await uploadDropFile(apiOptions, deployId, fileObj.normalizedPath, body, token)
         return
       } catch (error) {
-        lastError = error as DropApiError
-        if (lastError.status === 400 || lastError.status === 422) {
+        const isPermanentFailure =
+          error instanceof Error && 'status' in error && (error.status === 400 || error.status === 422)
+        if (isPermanentFailure || attempt + 1 > maxRetry) {
           throw error
         }
-        if (attempt < maxRetry) {
-          await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1)))
-        }
+        await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1)))
       }
     }
-    if (lastError) {
-      throw lastError
-    }
   }
 
   await pMap(uploadList, uploadFile, { concurrency: concurrentUpload })

src/utils/deploy/upload-files.ts

@@ -231,8 +231,9 @@ const retryUpload = <T>(uploadFn: (retryCount: number) => Promise<T>, maxRetry:
       // user the delay before next reconnection attempt.
     })
 
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises -- FIXME: the listener's promise is discarded
-    fibonacciBackoff.on('ready', tryUpload)
+    fibonacciBackoff.on('ready', (retryIndex) => {
+      void tryUpload(retryIndex)
+    })
 
     fibonacciBackoff.on('fail', () => {
       reject(lastError)

src/utils/deploy/util.ts

@@ -12,8 +12,13 @@ export type Deploy = Awaited<ReturnType<NetlifyAPI['getSiteDeploy']>>
 export type ReadyDeploy = Deploy &
   Required<Pick<Deploy, 'id' | 'site_id' | 'name' | 'admin_url' | 'url' | 'ssl_url' | 'deploy_url' | 'deploy_ssl_url'>>
 
-interface DeployStateError extends Error {
-  deploy?: Deploy
+export class DeployStateError extends Error {
+  constructor(
+    message: string,
+    readonly deploy: Deploy,
+  ) {
+    super(message)
+  }
 }
 
 // normalize windows paths to unix paths
@@ -24,86 +29,51 @@ export const normalizePath = (relname: string): string => {
   return relname.split(sep).join('/')
 }
 
-// poll an async deployId until its done diffing
-export const waitForDiff = async (
+// https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33
+const DIFFED_STATES = new Set(['prepared', 'uploading', 'uploaded', 'ready'])
+const READY_STATES = new Set(['ready'])
+
+const pollDeployUntil = async (
   api: Pick<NetlifyAPI, 'getSiteDeploy'>,
   deployId: string,
   siteId: string,
   timeout: number,
-): Promise<Deploy> => {
-  const loadDeploy = async () => {
-    const siteDeploy = await api.getSiteDeploy({ siteId, deployId })
+  targetStates: Set<string>,
+): Promise<Deploy> =>
+  await pWaitFor(
+    async () => {
+      const siteDeploy = await api.getSiteDeploy({ siteId, deployId })
 
-    switch (siteDeploy.state) {
-      // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33
-      case 'error': {
-        const deployError: DeployStateError = new Error(siteDeploy.error_message || `Deploy ${deployId} had an error`)
-        deployError.deploy = siteDeploy
-        throw deployError
-      }
-      case 'prepared':
-      case 'uploading':
-      case 'uploaded':
-      case 'ready': {
-        return pWaitFor.resolveWith(siteDeploy)
-      }
-      case 'preparing':
-      default: {
-        return false
+      if (siteDeploy.state === 'error') {
+        throw new DeployStateError(siteDeploy.error_message || `Deploy ${deployId} had an error`, siteDeploy)
       }
-    }
-  }
 
-  const deploy = await pWaitFor(loadDeploy, {
-    interval: DEPLOY_POLL,
-    timeout: {
-      milliseconds: timeout,
-      message: 'Timeout while waiting for deploy',
+      return siteDeploy.state !== undefined && targetStates.has(siteDeploy.state) && pWaitFor.resolveWith(siteDeploy)
+    },
+    {
+      interval: DEPLOY_POLL,
+      timeout: {
+        milliseconds: timeout,
+        message: 'Timeout while waiting for deploy',
+      },
     },
-  })
+  )
 
-  return deploy
-}
+// poll an async deployId until its done diffing
+export const waitForDiff = async (
+  api: Pick<NetlifyAPI, 'getSiteDeploy'>,
+  deployId: string,
+  siteId: string,
+  timeout: number,
+): Promise<Deploy> => await pollDeployUntil(api, deployId, siteId, timeout, DIFFED_STATES)
 
 // Poll a deployId until its ready
 export const waitForDeploy = async (
   api: Pick<NetlifyAPI, 'getSiteDeploy'>,
   deployId: string,
   siteId: string,
   timeout: number,
-): Promise<ReadyDeploy> => {
-  const loadDeploy = async () => {
-    const siteDeploy = await api.getSiteDeploy({ siteId, deployId })
-    switch (siteDeploy.state) {
-      // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33
-      case 'error': {
-        const deployError: DeployStateError = new Error(siteDeploy.error_message || `Deploy ${deployId} had an error`)
-        deployError.deploy = siteDeploy
-        throw deployError
-      }
-      case 'ready': {
-        return pWaitFor.resolveWith(siteDeploy)
-      }
-      case 'preparing':
-      case 'prepared':
-      case 'uploaded':
-      case 'uploading':
-      default: {
-        return false
-      }
-    }
-  }
-
-  const deploy = await pWaitFor(loadDeploy, {
-    interval: DEPLOY_POLL,
-    timeout: {
-      milliseconds: timeout,
-      message: 'Timeout while waiting for deploy',
-    },
-  })
-
-  return deploy as ReadyDeploy
-}
+): Promise<ReadyDeploy> => (await pollDeployUntil(api, deployId, siteId, timeout, READY_STATES)) as ReadyDeploy
 
 // Transform the fileShaMap and fnShaMap into a generic shaMap that file-uploader.js can use
 export const getUploadList = <T>(required: string[] | undefined, shaMap: Record<string, T[]> | undefined): T[] => {