Can't connect to docker containers started by "fabric8:docker-maven-plugin" during CI/CD

Hey community, recently tried porting an older maven project to CircleCI and I ran into some issues while running integration tests. I have the project set up to integration tests after it starts the appropriate docker containers. The problem that I am running into is that after the docker containers are started, I can’t connect to the containers to execute requests. Does anyone see anything wrong here?

I have the relevant docker-maven-plugin and circleci.yml configuration as well as the output from a docker inspect.

Docker Maven Plugin Config

<properties>
  <!-- Integration Test Variables -->
  <bind.host.ip>${it.docker.service.host}</bind.host.ip>
  <it.docker.database.port>3100</it.docker.database.port>
  <it.docker.postgres.version>9.5.4</it.docker.postgres.version>
  <it.docker.service.host>172.18.0.2</it.docker.service.host>
  <it.docker.service.port>3000</it.docker.service.port>
  <it.docker.network>${project.artifactId}</it.docker.network>
  <it.docker.host.path>${project.basedir}</it.docker.host.path>
</properties>

<plugin>
  <groupId>io.fabric8</groupId>
  <artifactId>docker-maven-plugin</artifactId>
  <extensions>true</extensions>
  <executions>
    <execution>
      <id>default-start</id>
      <phase>pre-integration-test</phase>
      <goals>
        <goal>start</goal>
      </goals>
    </execution>
    <execution>
      <id>default-stop</id>
      <phase>post-integration-test</phase>
      <goals>
        <goal>stop</goal>
      </goals>
    </execution>
    <execution>
      <id>default-remove</id>
      <phase>validate</phase>
    </execution>
    <execution>
      <id>default-build</id>
      <phase>package</phase>
      <goals>
        <goal>build</goal>
      </goals>
    </execution>
    <execution>
      <id>default-push</id>
      <phase>deploy</phase>
      <goals>
        <goal>push</goal>
      </goals>
    </execution>
  </executions>
  <configuration>
    <!--<registry>docker.pkg.github.com</registry>-->
    <dockerHost>${it.docker.host}</dockerHost>
    <autoCreateCustomNetworks>true</autoCreateCustomNetworks>
    <removeAll>true</removeAll>
    <showLogs>true</showLogs>
    <images>
      <image>
        <name>${project.artifactId}:${project.version}</name>
        <alias>${project.artifactId}</alias>
        <build>
          <contextDir>${project.basedir}</contextDir>
          <dockerFile>${project.basedir}/src/main/docker/Dockerfile</dockerFile>
          <args>
            <version>${project.version}</version>
            <service_name>${project.artifactId}</service_name>
          </args>
        </build>
        <run>
          <env combine.children="append">
            <!-- Hydrate environment variables here -->
            <APPLICATION_NAME>${project.artifactId}</APPLICATION_NAME>
            <APPLICATION_PORT>${it.docker.service.port}</APPLICATION_PORT>
          </env>
          <ports>
            <port>${it.docker.service.port}:${it.docker.service.port}/tcp</port>
            <port>+host.ip:${it.docker.service.port}:${it.docker.service.port}/tcp</port>
          </ports>
          <network>
            <mode>custom</mode>
            <name>aperture</name>
          </network>
          <dependsOn>
            <container>${project.artifactId}-database</container>
          </dependsOn>
          <volumes>
            <bind>
              <volume>${it.docker.host.path}/logs:/opt/${project.artifactId}/logs</volume>
            </bind>
          </volumes>
          <wait>
            <http>
              <url>http://${it.docker.service.host}:${it.docker.service.port}/healthcheck</url>
              <method>GET</method>
              <status>200</status>
            </http>
            <time>10000</time>
            <kill>1000</kill>
            <shutdown>500</shutdown>
          </wait>
        </run>
      </image>
      <image>
        <name>postgres:${it.docker.postgres.version}</name>
        <alias>${project.artifactId}-database</alias>
        <run>
          <ports>
            <port>${it.docker.database.port}:${it.docker.database.port}</port>
          </ports>
          <network>
            <mode>custom</mode>
            <name>aperture</name>
          </network>
          <wait>
            <log>
              (?s)database system is ready to accept connections.*database system is ready to accept connections
            </log>
            <time>10000</time>
          </wait>
        </run>
      </image>
    </images>
  </configuration>
</plugin>

CircleCI Yaml

version: 2
jobs:
  build: # runs not using Workflows must have a `build` job as entry point

    working_directory: ~/aperture # directory where steps will run

    docker: # run the steps with Docker
      - image: circleci/openjdk:8-jdk-stretch # ...with this image as the primary container; this is where all `steps` will run

    environment:
      DOCKER_HOST: tcp://127.0.0.1:2376
      DOCKER_CONTAINER_HOST: 172.18.0.2
      TT_JWT_ISSUER: identity-service
      TT_JWT_SECRET: fake-secret

    steps: # a collection of executable commands

      - setup_remote_docker

      - checkout # check out source code to working directory

      - run: mvn verify

Docker Inspect

[
    {
        "Id": "225696309efe495af596f9a3085b09dad1b545ef8a299340d52fb1831242f83d",
        "Created": "2020-05-05T05:02:20.518847682Z",
        "Path": "/bin/sh",
        "Args": [
            "-c",
            "java -jar /opt/${APPLICATION_NAME}/${APPLICATION_NAME}.jar"
        ],
        "State": {
            "Status": "running",
            "Running": true,
            "Paused": false,
            "Restarting": false,
            "OOMKilled": false,
            "Dead": false,
            "Pid": 4233,
            "ExitCode": 0,
            "Error": "",
            "StartedAt": "2020-05-05T05:02:20.967627741Z",
            "FinishedAt": "0001-01-01T00:00:00Z"
        },
        "Image": "sha256:3381da9cd83e7e7ecdc4f935d2c9d502622fa71d918f8aed95be70a4ce16ad4a",
        "ResolvConfPath": "/var/lib/docker/containers/225696309efe495af596f9a3085b09dad1b545ef8a299340d52fb1831242f83d/resolv.conf",
        "HostnamePath": "/var/lib/docker/containers/225696309efe495af596f9a3085b09dad1b545ef8a299340d52fb1831242f83d/hostname",
        "HostsPath": "/var/lib/docker/containers/225696309efe495af596f9a3085b09dad1b545ef8a299340d52fb1831242f83d/hosts",
        "LogPath": "/var/lib/docker/containers/225696309efe495af596f9a3085b09dad1b545ef8a299340d52fb1831242f83d/225696309efe495af596f9a3085b09dad1b545ef8a299340d52fb1831242f83d-json.log",
        "Name": "/identity-service-1",
        "RestartCount": 0,
        "Driver": "aufs",
        "Platform": "linux",
        "MountLabel": "",
        "ProcessLabel": "",
        "AppArmorProfile": "docker-default",
        "ExecIDs": null,
        "HostConfig": {
            "Binds": [
                "/home/circleci/aperture/identity-service/logs:/opt/identity-service/logs"
            ],
            "ContainerIDFile": "",
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "NetworkMode": "aperture",
            "PortBindings": {
                "3000/tcp": [
                    {
                        "HostIp": "",
                        "HostPort": "3000"
                    }
                ]
            },
            "RestartPolicy": {
                "Name": "",
                "MaximumRetryCount": 0
            },
            "AutoRemove": false,
            "VolumeDriver": "",
            "VolumesFrom": [],
            "CapAdd": null,
            "CapDrop": null,
            "Dns": null,
            "DnsOptions": null,
            "DnsSearch": null,
            "ExtraHosts": null,
            "GroupAdd": null,
            "IpcMode": "shareable",
            "Cgroup": "",
            "Links": null,
            "OomScoreAdj": 0,
            "PidMode": "",
            "Privileged": false,
            "PublishAllPorts": false,
            "ReadonlyRootfs": false,
            "SecurityOpt": null,
            "UTSMode": "",
            "UsernsMode": "",
            "ShmSize": 67108864,
            "Runtime": "runc",
            "ConsoleSize": [
                0,
                0
            ],
            "Isolation": "",
            "CpuShares": 0,
            "Memory": 0,
            "NanoCpus": 0,
            "CgroupParent": "",
            "BlkioWeight": 0,
            "BlkioWeightDevice": null,
            "BlkioDeviceReadBps": null,
            "BlkioDeviceWriteBps": null,
            "BlkioDeviceReadIOps": null,
            "BlkioDeviceWriteIOps": null,
            "CpuPeriod": 0,
            "CpuQuota": 0,
            "CpuRealtimePeriod": 0,
            "CpuRealtimeRuntime": 0,
            "CpusetCpus": "",
            "CpusetMems": "",
            "Devices": null,
            "DeviceCgroupRules": null,
            "DiskQuota": 0,
            "KernelMemory": 0,
            "MemoryReservation": 0,
            "MemorySwap": 0,
            "MemorySwappiness": null,
            "OomKillDisable": false,
            "PidsLimit": 0,
            "Ulimits": null,
            "CpuCount": 0,
            "CpuPercent": 0,
            "IOMaximumIOps": 0,
            "IOMaximumBandwidth": 0
        },
        "GraphDriver": {
            "Data": null,
            "Name": "aufs"
        },
        "Mounts": [
            {
                "Type": "bind",
                "Source": "/home/circleci/aperture/identity-service/logs",
                "Destination": "/opt/identity-service/logs",
                "Mode": "",
                "RW": true,
                "Propagation": "rprivate"
            }
        ],
        "Config": {
            "Hostname": "225696309efe",
            "Domainname": "",
            "User": "",
            "AttachStdin": false,
            "AttachStdout": false,
            "AttachStderr": false,
            "ExposedPorts": {
                "3000/tcp": {}
            },
            "Tty": false,
            "OpenStdin": false,
            "StdinOnce": false,
            "Env": [
                "APPLICATION_PORT=3000",
                "APPLICATION_NAME=identity-service",
                "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
                "LANG=en_US.UTF-8",
                "LANGUAGE=en_US:en",
                "LC_ALL=en_US.UTF-8",
                "JAVA_HOME=/usr/lib/jvm/zulu11-ca",
                "IO_THREADS=2",
                "WORKER_THREADS=16"
            ],
            "Cmd": [
                "/bin/sh",
                "-c",
                "java -jar /opt/${APPLICATION_NAME}/${APPLICATION_NAME}.jar"
            ],
            "ArgsEscaped": true,
            "Image": "identity-service:0.1.0-SNAPSHOT",
            "Volumes": {
                "/opt/identity-service/logs": {}
            },
            "WorkingDir": "/opt/identity-service",
            "Entrypoint": null,
            "OnBuild": null,
            "Labels": {
                "dmp.coordinates": "com.trinitytap:identity-service:0.1.0-SNAPSHOT"
            }
        },
        "NetworkSettings": {
            "Bridge": "",
            "SandboxID": "63b9d3f999ed03a000e8d0157a51d2e0769d16a153f4812eb1756ed5d703a9cb",
            "HairpinMode": false,
            "LinkLocalIPv6Address": "",
            "LinkLocalIPv6PrefixLen": 0,
            "Ports": {
                "3000/tcp": [
                    {
                        "HostIp": "0.0.0.0",
                        "HostPort": "3000"
                    }
                ]
            },
            "SandboxKey": "/var/run/docker/netns/63b9d3f999ed",
            "SecondaryIPAddresses": null,
            "SecondaryIPv6Addresses": null,
            "EndpointID": "",
            "Gateway": "",
            "GlobalIPv6Address": "",
            "GlobalIPv6PrefixLen": 0,
            "IPAddress": "",
            "IPPrefixLen": 0,
            "IPv6Gateway": "",
            "MacAddress": "",
            "Networks": {
                "aperture": {
                    "IPAMConfig": null,
                    "Links": null,
                    "Aliases": [
                        "225696309efe"
                    ],
                    "NetworkID": "ecaefc35fd608ee6de40ec07a6cfb8655c47c4c03d480acf8e2fe6a4adf823bb",
                    "EndpointID": "dbca3e1209fef071cdc8478156e5313448024f203d4f73912f204e4c936f2276",
                    "Gateway": "172.18.0.1",
                    "IPAddress": "172.18.0.3",
                    "IPPrefixLen": 16,
                    "IPv6Gateway": "",
                    "GlobalIPv6Address": "",
                    "GlobalIPv6PrefixLen": 0,
                    "MacAddress": "02:42:ac:12:00:03",
                    "DriverOpts": null
                }
            }
        }
    }
]

Spent some more time familiarizing myself with the CircleCI platform. Turns out to fix this problem we needed run this on a ‘machine’ executor instead of ‘docker’. This would give us access to a true VM and thus access to the privileged containers that are running during build time.