Docker-compose how to cache dependencies?

I currently have this step on my config, it installs my dependencies and save then on yarn’s cache folder:

install-dependencies:
    executor: node/default
    steps:
      - checkout
      - restore_cache:
          keys:
            - dependencies-{{ checksum "yarn.lock" }}

      - run: yarn install

      - save_cache:
          paths:
            - ~/.cache/yarn
          key: dependencies-{{ checksum "yarn.lock" }}

All working good except when my e2e tests run using docker-compose. It builds my image and always install everything since no dependency cache is restored. Is it possible to restore cache in this scenario?

My e2e job without the restore_cache step:

tests-e2e:
    executor: node/default
    steps:
      - checkout

      - setup_remote_docker:
          docker_layer_caching: true

      - run: docker-compose build test-api-e2e

      - run: make test-e2e

Dockerfile:

FROM node:12.16.1-alpine

WORKDIR /app

ADD / ./
RUN yarn install # I want my dependency cache to be restored here

CMD ["npm", "start"]

Docker-compose:

version: '3.3'
services:
  dynamodb:
    image: amazon/dynamodb-local:1.12.0
    hostname: dynamodb
    ports:
      - "8000"

  test-api-e2e: # run when make test-e2e is used
    build: .
    command: npm run test:e2e
    depends_on:
      - dynamodb
    links:
      - dynamodb