安装

Linux

在线安装

首先从github 上下载 https://github.com/docker/compose/releases

下载 docker-compose-linux-x86_64

上传到linux中

mv docker-compose-linux-x86_64 /usr/local/bin/
cd /usr/local/bin/
mv docker-compose-linux-x86_64 docker-compose
chmod +x /usr/local/bin/docker-compose
ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

检查是否安装成功

docker-compose version

离线安装 - 可能会出现下载问题

运行以下命令以下载 Docker Compose 的当前稳定版本：

sudo curl -L "https://github.com/docker/compose/releases/download/v2.2.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

将可执行权限应用于二进制文件：

sudo chmod +x /usr/local/bin/docker-compose

创建软链：

sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose

测试是否安装成功：

docker-compose version

集群搭建

Flink 集群

首先启动flink 容器 JobManager、TaskManager 两个容器将配置文件复制出来方便挂载

docker network create flink-network

docker run \
  -itd \
  --name=jobmanager \
  --publish 8081:8081 \
  --network flink-network \
  --env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \
  flink:1.16.0-scala_2.12-java8 jobmanager 

docker run \
  -itd \
  --name=taskmanager \
  --network flink-network \
  --env FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" \
  flink:1.16.0-scala_2.12-java8 taskmanager

创建本地卷挂载目录，拷贝文件

 mkdir -p /usr/local/flink-docker/jobmanager
 mkdir -p /usr/local/flink-docker/taskmanager
docker cp jobmanager:/opt/flink/lib /usr/local/flink-docker/jobmanager
docker cp jobmanager:/opt/flink/log /usr/local/flink-docker/jobmanager
docker cp jobmanager:/opt/flink/conf /usr/local/flink-docker/jobmanager

docker cp taskmanager:/opt/flink/lib /usr/local/flink-docker/taskmanager
docker cp taskmanager:/opt/flink/log /usr/local/flink-docker/taskmanager
docker cp taskmanager:/opt/flink/conf /usr/local/flink-docker/taskmanager

搭建集群
docker-compose.yaml 文件

version: "2.2"
services:
  jobmanager:
    image: flink:1.16.0-scala_2.12-java8
    container_name: jobmanager-1
    expose:
      - "6123"
    ports:
      - "8081:8081"
    command: jobmanager
    volumes:
      - /usr/local/flink-docker/jobmanager/conf:/opt/flink/conf
      - /usr/local/flink-docker/jobmanager/lib:/opt/flink/lib
      - /usr/local/flink-docker/jobmanager/log:/opt/flink/log
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        parallelism.default: 2
          #web.upload.dir: /opt/flink/target
    networks:
      - flink-network
  taskmanager:
    image: flink:1.16.0-scala_2.12-java8
    container_name: taskmanager-1
    depends_on:
      - jobmanager
    command: taskmanager
    scale: 1
    volumes:
      - /usr/local/flink-docker/taskmanager/conf:/opt/flink/conf
      - /usr/local/flink-docker/taskmanager/lib:/opt/flink/lib
      - /usr/local/flink-docker/taskmanager/log:/opt/flink/log
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 8
        parallelism.default: 2
    networks:
      - flink-network
networks:
  flink-network:
    external: true

启动集群
docker-compose up -d

Kafka 集群

docker network create kfk-network

docker -compose.yaml

services:
  kafka1:
    image: 'bitnami/kafka:3.6.1'
    ports:
      - '19092:19092'
    environment:
      - KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka1:9092, EXTERNAL://<宿主机IP>:19092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
    networks:
      - kfk-network
  kafka2:
    image: 'bitnami/kafka:3.6.1'
    ports:
      - '29092:19092'
    environment:
      - KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka2:9092, EXTERNAL://<宿主机IP>:29092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
    networks:
      - kfk-network
  kafka3:
    image: 'bitnami/kafka:3.6.1'
    ports:
      - '39092:19092'
    environment:
      - KAFKA_KRAFT_CLUSTER_ID=EX5bq5NfRe2IX1nhxrSO6g
      - KAFKA_CFG_NODE_ID=3
      - KAFKA_CFG_PROCESS_ROLES=broker,controller
      - KAFKA_CFG_LISTENERS=INTERNAL://:9092, EXTERNAL://:19092, CONTROLLER://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka3:9092, EXTERNAL://<宿主机IP>:39092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT, EXTERNAL:PLAINTEXT, CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=1@kafka1:9093, 2@kafka2:9093, 3@kafka3:9093
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
    networks:
      - kfk-network
  kafka-ui:
    image: provectuslabs/kafka-ui:master
    container_name: kafka-ui
    ports:
      - "38080:8080"
    restart: always
    environment:
      - KAFKA_CLUSTERS_0_NAME=local
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka1:9092,kafka2:9092,kafka3:9092
      - KAFKA_CLUSTERS_0_READONLY=true
    depends_on:
      - kafka1
      - kafka2
      - kafka3
    networks:
      - kfk-network
networks:
  kfk-network:
    external: true