docker-compose部署MinIO服务器单点集群, Nginx进行反向代理

docker-compose部署MinIO服务器单点集群, Nginx进行反向代理

艾瑞斯胡 1,014 2022-01-01

一、引导语

前面介绍了MinIO的一些特点,相信大家对MinIO有了一个初步的认识。下面介绍一下,MinIO的安装。

MinIO 对Kubernetes是非常友好的,在Kubernets上的安装,将在后续的文章中介绍。

这里介绍如何在docker上使用docker-compose进行部署,并使用nginx进行https代理访问

二、使用Nginx反向代理MinIO

1. 创建docker证书

说明:创建证书的目录必须是/root/docker-certs

# mkdir -p /root/docker-certs
# cd /root/docker-certs

docker-server.cnf

# OpenSSL to generate a certificate signing requests(csr) configuration file.
# v1

[ req ]
# Options for the `req` tool (`man req`).
# use prompt config control user interactive
prompt = yes

default_bits        = 4096
distinguished_name  = req_distinguished_name
string_mask         = utf8only

# SHA-1 is deprecated, so use SHA-2 instead.
default_md          = sha256

# Extension to add when the -x509 option is used.
#x509_extensions     = v3_ca
req_extensions     = v3_req

[ req_distinguished_name ]
# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
countryName            			= Country Name (two letter code)
countryName            			= CN
stateOrProvinceName 			= State or Province Name (full name)
stateOrProvinceName_default     = Shanghai
localityName 					= Locality Name (eg, city)
localityName_default	        = SH
organizationName                = Organization Name
organizationName_default        = My Group
organizationalUnitName          = Department Name
organizationalUnitName_default  = Java Development Team 
commonName 						= Common Name (eg, YOUR name)
commonName_default				= 
emailAddress 					= Email Address
emailAddress_default            = private

generate-tool.sh

说明:

  • 在linux下运行以下脚本之前,要让脚本可执行

    # chmod +x ./generate-tool.sh
    
  • 在生成的过程中,遇到key要求密码,请输入:p@ssw0rd,最后导出密码是:12123123

  • 脚本运行时,会要求输入域名,请填写你需要的域名,比如:docker.homeserver.net

  • 证书创建好后,需要在你的开发主机的host中添加域名映射,左边的IP地址是你自己的docker所在虚拟机的IP地址

    192.168.3.26 docker.dooool.com
    
    
  • 将证书拷贝到指定目录下

    因为docker证书对一台服务器的docker全局使用,所以拷贝到通用的位置去

    # \cp docker-cert.pem docker-key.pem ca.pem /home/ws/config/ssl/docker/
    
#!/bin/bash

read -e -p "Please enter the domain name? " HOST

ROOT_DIR=/root/docker-certs
CURRENT_DATETIME=`date +%Y-%m-%d_%H:%M:%S`
SAVE_DIR=$ROOT_DIR/$CURRENT_DATETIME/$HOST
PKCS12_EXPORT_PASSWORD=12123123
KEY_PASSWORD=p@ssw0rd

DOCKER_CA_KEY_FILE=$SAVE_DIR/ca-key.pem
DOCKER_CA_CERT_FILE=$SAVE_DIR/ca.pem
DOCKER_CA_P12_FILE=$SAVE_DIR/ca.p12

DOCKER_SERVER_CNF_FILE=$SAVE_DIR/docker.cnf
DOCKER_SERVER_CONFIG_FILE=$ROOT_DIR/docker-server.cnf
DOCKER_SERVER_KEY_FILE=$SAVE_DIR/docker-key.pem
DOCKER_SERVER_CSR_FILE=$SAVE_DIR/docker-req.csr
DOCKER_SERVER_CERT_FILE=$SAVE_DIR/docker-cert.pem
DOCKER_SERVER_P12_FILE=$SAVE_DIR/docker-cert.p12

DOCKER_CLIENT_CNF_FILE=$SAVE_DIR/docker-client.cnf
DOCKER_CLIENT_KEY_FILE=$SAVE_DIR/docker-client-key.pem
DOCKER_CLIENT_CSR_FILE=$SAVE_DIR/docker-client-req.csr
DOCKER_CLIENT_CERT_FILE=$SAVE_DIR/docker-client-cert.pem
DOCKER_CLIENT_P12_FILE=$SAVE_DIR/docker-client-cert.p12

if [ $HOST != "" ]; then
	# read variables from user
	read -e -p "enter host ip:" HOST_IP
	read -e -p "Enter docker client name: " DOCKER_CLIENT_NAME

	echo -e "\n"
	# create directory
	echo -e "\n>> create save directory"
	mkdir -p $SAVE_DIR

	echo -e "\n>>create ca certificate"
	# 生成ca键
	openssl genrsa -aes256 -out $DOCKER_CA_KEY_FILE 4096;
	# 生成ca证书
	openssl req -subj "/C=CN/ST=Shanghai/L=SH/O=My\ Group/OU=Java\ Development\ Team/CN=$HOST" -new -x509 -days 365 -passin pass:p@ssw0rd -key $DOCKER_CA_KEY_FILE -sha256 -out $DOCKER_CA_CERT_FILE;

	echo -e "\n>> create server certificate"
	# 生成服务器私钥
	openssl genrsa -out $DOCKER_SERVER_KEY_FILE 4096;
	# 生成服务端证书配置文件
	#echo subjectAltName = DNS:$HOST,IP:192.168.0.60,IP:139.224.247.222 > $CNF_FILE;
	echo subjectAltName = DNS:$HOST,IP:$HOST_IP >> $DOCKER_SERVER_CNF_FILE;
	echo extendedKeyUsage = serverAuth >> $DOCKER_SERVER_CNF_FILE;
	# 生成服务器证书请求
	openssl req -subj "/CN=$HOST" -sha256 -new -key $DOCKER_SERVER_KEY_FILE -out $DOCKER_SERVER_CSR_FILE;
	#openssl req -subj "/C=CN/ST=Shanghai/L=SH/O=My\ Group/OU=Java\ Development\ Team/CN=$HOST" -sha256 -new -key $DOCKER_SERVER_KEY_FILE -out $DOCKER_SERVER_CSR_FILE;
	#openssl req -config $DOCKER_SERVER_CONFIG_FILE -key $DOCKER_SERVER_KEY_FILE -new -sha256 -out $DOCKER_SERVER_CSR_FILE;
	# 使用CA证书签发服务器证书
	openssl x509 -req -days 365 -sha256 -in $DOCKER_SERVER_CSR_FILE -passin pass:p@ssw0rd -CA $DOCKER_CA_CERT_FILE -CAkey $DOCKER_CA_KEY_FILE -CAcreateserial -out $DOCKER_SERVER_CERT_FILE -extfile $DOCKER_SERVER_CNF_FILE

	echo -e "\n>> create client certificate\n"
	# 生成客户端私钥
	openssl genrsa -out $DOCKER_CLIENT_KEY_FILE 4096
	# 生成客户端证书请求
	openssl req -subj "/CN=$DOCKER_CLIENT_NAME" -key $DOCKER_CLIENT_KEY_FILE -new -sha256 -out $DOCKER_CLIENT_CSR_FILE;
	# 生成客户端证书配置文件
	echo extendedKeyUsage = clientAuth >> $DOCKER_CLIENT_CNF_FILE
	# 使用CA证书签发客户端证书
	openssl x509 -req -days 365 -sha256 -in $DOCKER_CLIENT_CSR_FILE -passin pass:p@ssw0rd -CA $DOCKER_CA_CERT_FILE -CAkey $DOCKER_CA_KEY_FILE -CAcreateserial -out $DOCKER_CLIENT_CERT_FILE -extfile $DOCKER_CLIENT_CNF_FILE

	echo -e "\n>> export certificates to p12 format\n"
	# 导出证书
	echo -e "export $DOCKER_CA_CERT_FILE"
	openssl pkcs12 -export -clcerts -in $DOCKER_CA_CERT_FILE -password pass:12123123 -passin pass:p@ssw0rd -inkey $DOCKER_CA_KEY_FILE -out $DOCKER_CA_P12_FILE ;
	echo -e "\nexport $DOCKER_SERVER_CERT_FILE"
	openssl pkcs12 -export -clcerts -in $DOCKER_SERVER_CERT_FILE -password pass:12123123 -inkey $DOCKER_SERVER_KEY_FILE -out $DOCKER_SERVER_P12_FILE;
	echo -e "\nexport $DOCKER_CLIENT_CERT_FILE"
	openssl pkcs12 -export -clcerts -in $DOCKER_CLIENT_CERT_FILE -password pass:12123123 -inkey $DOCKER_CLIENT_KEY_FILE -out $DOCKER_CLIENT_P12_FILE;

	echo -e "\n>> change privileges of keys and certificates\n"
	# 修改证书权限
	chmod -v 0400 $DOCKER_CA_KEY_FILE $DOCKER_SERVER_KEY_FILE $DOCKER_CLIENT_KEY_FILe
	chmod -v 0444 $DOCKER_CA_CERT_FILE $DOCKER_SERVER_CERT_FILE $DOCKER_CLIENT_CERT_FILE

	echo -e "\n 【Finished】"
	
else
	echo "You should provide the domain name!"
fi

2. 编写docker-compose.yml文件

说明:

  • 以下文件使用了yaml文件的继承特性,将公有属性提取了出来,放在了 x-minio-common下,并将地址放在了&minio-common

变量中,然后使用*minio-common获取继承的内容,具体请参考docker-compose官方说明

version: '3.7'

# Settings and configurations that are common for all minio containers
x-minio-common: &minio-common
  image: minio/minio:$MINIO_VERSION
  command: server --address "0.0.0.0:9000" --console-address "0.0.0.0:9001" http://minio{1...4}/data{1...2}
  restart: always
  expose:
    - "9000"
    - "9001"
  environment:
    TZ: UTC
    MINIO_ROOT_USER: $MINIO_ROOT_USER
    MINIO_ROOT_PASSWORD: $MINIO_ROOT_PASSWORD
  extra_hosts:
    - $DOCKER_DOMAIN_HOST_NAME:$DOCKER_DOMAIN_HOST_IP
  networks:
    - minio-net
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 3

# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
# 定义了4个节点, 每个节点挂载2个卷(类似磁盘)
services:
  minio1:
    << : *minio-common
    hostname: minio1
    volumes:
      - data1-1:/data1
      - data1-2:/data2

  minio2:
    << : *minio-common
    hostname: minio2
    volumes:
      - data2-1:/data1
      - data2-2:/data2

  minio3:
    << : *minio-common
    hostname: minio3
    volumes:
      - data3-1:/data1
      - data3-2:/data2

  minio4:
    << : *minio-common
    hostname: minio4
    volumes:
      - data4-1:/data1
      - data4-2:/data2

  nginx:
    image: nginx:1.19.2-alpine
    hostname: nginx
    volumes:
      - ./conf/nginx.conf:/etc/nginx/nginx.conf:ro
      - /home/ws/config/ssl/docker/docker-cert.pem:/etc/nginx/certs/public.crt
      - /home/ws/config/ssl/docker/docker-key.pem:/etc/nginx/certs/private.key
    networks:
      - minio-net
    ports:
      - "49000:9000"
      - "49001:9001"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4
      
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
## 下面创建挂载卷, 每个挂在卷都是挂在到docker所在主机的磁盘目录
volumes:
  data1-1:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data1-1
  data1-2:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data1-2
  data2-1:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data2-1
  data2-2:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data2-2
  data3-1:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data3-1
  data3-2:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data3-2
  data4-1:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data4-1
  data4-2:
    driver_opts:
      type: ext4
      o: bind
      device: /home/ws/docker/minio/datas/data4-2

# define minionet to create
networks:
  minio-net:
    name: minionet
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.30.6.0/24
          gateway: 172.30.6.1
      

3. 创建环境变量文件 .env

上面的docker-compose文件使用了环境变量,所以需要创建以下的环境变量文件,放在与docker-compose.yml同级的目录下

DOCKER_DOMAIN_HOST_NAME=docker.homeserver.net
DOCKER_DOMAIN_HOST_IP=192.168.3.26

MINIO_VERSION=latest
MINIO_ROOT_USER=AKIAIOSFODNN7EXAPPLE
MINIO_ROOT_PASSWORD=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAPPLEKEY

4. 编写 nginx.conf 文件

nginx.conf 文件放在当前minio目录下的conf目录下

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       		/etc/nginx/mime.types;
    default_type  		application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  		/var/log/nginx/access.log  main;
    sendfile        	on;
    keepalive_timeout  	65;
    
    # enable compress
    gzip  on; 
  
    #自定义变量 $connection_upgrade                                                                                                   
    map $http_upgrade $connection_upgrade {
      default          keep-alive;  #默认为keep-alive 可以支持 一般http请求
      'websocket'      upgrade;     #如果为websocket 则为 upgrade 可升级的。
    } 

    # include /etc/nginx/conf.d/*.conf;
	# minio api loadbalancer
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
        server minio4:9000;
    }

    # minio console loadbalancer
    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
        server minio4:9001;
    }

    server {
        listen       9000 ssl;
        listen  [::]:9000 ssl;
        server_name  localhost;
		
        # define ssl certificates
        ssl_certificate 		/etc/nginx/certs/public.crt;
        ssl_certificate_key		/etc/nginx/certs/private.key;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 1000m;
        # To disable buffering
        proxy_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }

    server {
        listen       9001 ssl;
        listen  [::]:9001 ssl;
        server_name  localhost;

        # define ssl certificates
        ssl_certificate 		/etc/nginx/certs/public.crt;
        ssl_certificate_key		/etc/nginx/certs/private.key;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 1000m;
        # To disable buffering
        proxy_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;

            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;

            proxy_connect_timeout 300;
            
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            
            chunked_transfer_encoding off;

            proxy_pass http://console;
        }
    }
}

5.创建两个脚本文件

create-dir.sh

目的:用于创建目录

#!/bin/bash

mkdir -p ./datas/data1-1 ./datas/data1-2 ./datas/data2-1 ./datas/data2-2 ./datas/data3-1 ./datas/data3-2 ./datas/data4-1 ./datas/data4-2;

operation.sh

目的:启动容器

#!/bin/bash

docker-compose --env-file ./.env -f ./docker-compose.yml $1 $2;

6. 启动MinIO服务器实例

# ./operation.sh up -d

7. 开启防火墙端口

# firewall-cmd --zone=public --add-port=49000/tcp --permanent
# firewall-cmd --zone=public --add-port=49001/tcp --permanent
# firewall-cmd --reload
# firewall-cmd --list-all

# OSS # MinIO # 对象存储 # 集群搭建