cdk synth gets stuck at AssetStaging inside of DockerImageAsset

Question

My cdk configuration is

import * as cdk from 'aws-cdk-lib';
import { aws_s3 as s3, aws_lambda as lambda, aws_apigateway as apigateway, aws_ecr_assets as ecr_assets } from 'aws-cdk-lib';
import { Construct } from 'constructs';
import * as path from 'path';
// import * as sqs from 'aws-cdk-lib/aws-sqs';

/** Root directory of the entire project (not just the CDK project) */
const ROOT_DIR = path.resolve(__dirname, '../../../');
const DOCKER_DIR = path.join(ROOT_DIR, 'docker_images');
console.log('ROOT_DIR=',ROOT_DIR);

export class AppStack extends cdk.Stack {
  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
    super(scope, id, props);

    const packaged_blender_files = new s3.Bucket(this, 'packaged-blender-files', {
      versioned: true
    });
    console.log('Building docker');
    const package_scene_ecr_img = new ecr_assets.DockerImageAsset(this, 'package-scene', {
      directory: ROOT_DIR,
      file: 'docker_images/package_scene/Dockerfile',
      invalidation: {
        buildArgs: false,
      },

    });
    console.log('Building function');
    const handler = new lambda.DockerImageFunction(this, "package-scene", {
      code: lambda.DockerImageCode.fromEcr(package_scene_ecr_img.repository)

    });
  }
}

When I run cdk synth -vvv --debug what I see is

(bunch of logs above)
ROOT_DIR= C:\Users\Aleksander\source\repos\Rust\gmai_cad
Building docker

and it seems to get stuck at DockerImageAsset. After running debugger I noticed that this is the call stack where the function gets stuck

_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
_processFileOrDirectory (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
fingerprint (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\fingerprint.js:2)
fingerprint (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\fs\index.js:1)
calculateHash (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\asset-staging.js:2)
stageByCopying (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\asset-staging.js:1)
stageThisAsset (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\asset-staging.js:1)
obtain (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\private\cache.js:1)
AssetStaging (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\core\lib\asset-staging.js:1)
DockerImageAsset (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\aws-cdk-lib\aws-ecr-assets\lib\image-asset.js:2)
AppStack (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\lib\app-stack.ts:36)
<anonymous> (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\bin\app.ts:7)
Module._compile (internal/modules/cjs/loader:1105)
m._compile (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\ts-node\dist\index.js:857)
Module._extensions..js (internal/modules/cjs/loader:1159)
require.extensions.<computed> (c:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\node_modules\ts-node\dist\index.js:859)
Module.load (internal/modules/cjs/loader:981)
Module._load (internal/modules/cjs/loader:827)
executeUserEntryPoint (internal/modules/run_main:77)
<anonymous> (internal/main/run_main_module:17)

So the issue is that computing hash takes a long time when AssetStaging runs. What should I do about it? By the way, my project directory structure looks as follows:

PROJECT_ROOT
|- cdk
|   `- app
|       |- lib
|       |   `- app-stack.ts
|       `- ...
|- docker_images
|    `- package_scene
|          |- Dockerfile
|          |- assets (THIS DIR HAS MANY GBs IN IT)
|          |   `- ...
|          `- ...
` ...

Could it be that DockerImageAsset is trying to stage my entire project instead of just package_scene or perhaps compuing hashes of assets dir takes an eternity?

I also noticed there is infinite recursion during staging process. Keeps copying cdk.out into itself.

\\?\C:\Users\Aleksander\source\repos\Rust\gmai_cad\cdk\app\cdk.out\asset.d327e51061b3fc52319426d457c338523cac5acdab8e83c8e2edad9976850dc0\cdk\app\cdk.out\asset.168bdd25327ad31326b0c64be31b3c4013910dba73c03237b99cb863044814fa\cdk\app\cdk.out\asset.168bdd25327ad31326b0c64be31b3c4013910dba73c03237b99cb863044814fa\cdk\app\cdk.out\asset.168bdd25327ad31326b0c64be31b3c4013910dba73c03237b99cb863044814fa\cdk\app\cdk.out\asset.168bdd25327ad31326b0c64be31b3c4013910dba73c03237b99cb863044814fa\cdk\app\cdk.out

Answer 1

I found out that I should add exclude appropriately. This is actually a bug in cdk. The source code of DockerImageAsset uses

const cdkout = core_1.Stage.of(this)?.outdir ?? "cdk.out";

but it should really be something like

const cdkout = core_1.Stage.of(this)?.outdir ?? (PATH_TO_CDK+"cdk.out");

https://github.com/aws/aws-cdk/issues/3899

My solution looks like this

const ROOT_DIR = path.resolve(__dirname, '../../../');
const DOCKER_DIRNAME = 'docker_images';
const DOCKER_DIR = path.join(ROOT_DIR, DOCKER_DIRNAME);

function removeItemOnce<T>(arr:T[], value:T) {
  var index = arr.indexOf(value);
  if (index > -1) {
    arr.splice(index, 1);
  }
  return arr;
}
const EXCLUDE_FOR_DOCKER:string[] = fs.readdirSync(ROOT_DIR);
removeItemOnce(EXCLUDE_FOR_DOCKER, DOCKER_DIRNAME);
// remove here any other directories which you wish to include in docker
const DOCKER_DIR_CONTENTS:string[] = fs.readdirSync(DOCKER_DIR).map(x=>DOCKER_DIRNAME+'/'+x);


function excludes_for(docker_img:string):string[]{
  let arr = EXCLUDE_FOR_DOCKER.concat(DOCKER_DIR_CONTENTS);
  removeItemOnce(arr, DOCKER_DIRNAME+'/'+docker_img);
  return arr;
}
function docker_img_asset(scope: Construct, id: string, props: ecr_assets.DockerImageAssetProps):ecr_assets.DockerImageAsset{
  const img_dir = DOCKER_DIRNAME + '/' + id + '/';
  const p:ecr_assets.DockerImageAssetProps = {
    ...props,
    exclude:excludes_for(id).concat((props.exclude??[]).map(x=>img_dir+x)),
    file: img_dir + (props.file ?? 'Dockerfile'),
  };
  return new ecr_assets.DockerImageAsset(scope, id, p);
}

but it works for my specific directory structure. It should, however, be easy to adapt it to any other project structure.