一、引导语
前面介绍了MinIO的一些特点，相信大家对MinIO有了一个初步的认识。下面介绍一下，MinIO的安装。
MinIO 对Kubernetes是非常友好的，在Kubernets上的安装，将在后续的文章中介绍。
这里介绍如何在docker上使用docker-compose进行部署，并使用nginx进行https代理访问
二、使用Nginx反向代理MinIO
1. 创建docker证书
说明：创建证书的目录必须是： /root/docker-certs
# mkdir -p /root/docker-certs
# cd /root/docker-certs
docker-server.cnf
# OpenSSL to generate a certificate signing requests(csr) configuration file.
# v1
[ req ]
# Options for the `req` tool (`man req`).
# use prompt config control user interactive
prompt = yes
default_bits = 4096
distinguished_name = req_distinguished_name
string_mask = utf8only
# SHA-1 is deprecated, so use SHA-2 instead.
default_md = sha256
# Extension to add when the -x509 option is used.
#x509_extensions = v3_ca
req_extensions = v3_req
[ req_distinguished_name ]
# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
countryName = Country Name (two letter code)
countryName = CN
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_default = Shanghai
localityName = Locality Name (eg, city)
localityName_default = SH
organizationName = Organization Name
organizationName_default = My Group
organizationalUnitName = Department Name
organizationalUnitName_default = Java Development Team
commonName = Common Name (eg, YOUR name)
commonName_default =
emailAddress = Email Address
emailAddress_default = private
generate-tool.sh
说明：
-
在linux下运行以下脚本之前，要让脚本可执行
# chmod +x ./generate-tool.sh -
在生成的过程中，遇到key要求密码，请输入：
p@ssw0rd，最后导出密码是：12123123 -
脚本运行时，会要求输入域名，请填写你需要的域名，比如：docker.homeserver.net
-
证书创建好后，需要在你的开发主机的host中添加域名映射，左边的IP地址是你自己的docker所在虚拟机的IP地址
192.168.3.26 docker.dooool.com -
将证书拷贝到指定目录下
因为docker证书对一台服务器的docker全局使用，所以拷贝到通用的位置去
# \cp docker-cert.pem docker-key.pem ca.pem /home/ws/config/ssl/docker/
#!/bin/bash
read -e -p "Please enter the domain name? " HOST
ROOT_DIR=/root/docker-certs
CURRENT_DATETIME=`date +%Y-%m-%d_%H:%M:%S`
SAVE_DIR=$ROOT_DIR/$CURRENT_DATETIME/$HOST
PKCS12_EXPORT_PASSWORD=12123123
KEY_PASSWORD=p@ssw0rd
DOCKER_CA_KEY_FILE=$SAVE_DIR/ca-key.pem
DOCKER_CA_CERT_FILE=$SAVE_DIR/ca.pem
DOCKER_CA_P12_FILE=$SAVE_DIR/ca.p12
DOCKER_SERVER_CNF_FILE=$SAVE_DIR/docker.cnf
DOCKER_SERVER_CONFIG_FILE=$ROOT_DIR/docker-server.cnf
DOCKER_SERVER_KEY_FILE=$SAVE_DIR/docker-key.pem
DOCKER_SERVER_CSR_FILE=$SAVE_DIR/docker-req.csr
DOCKER_SERVER_CERT_FILE=$SAVE_DIR/docker-cert.pem
DOCKER_SERVER_P12_FILE=$SAVE_DIR/docker-cert.p12
DOCKER_CLIENT_CNF_FILE=$SAVE_DIR/docker-client.cnf
DOCKER_CLIENT_KEY_FILE=$SAVE_DIR/docker-client-key.pem
DOCKER_CLIENT_CSR_FILE=$SAVE_DIR/docker-client-req.csr
DOCKER_CLIENT_CERT_FILE=$SAVE_DIR/docker-client-cert.pem
DOCKER_CLIENT_P12_FILE=$SAVE_DIR/docker-client-cert.p12
if [ $HOST != "" ]; then
# read variables from user
read -e -p "enter host ip:" HOST_IP
read -e -p "Enter docker client name: " DOCKER_CLIENT_NAME
echo -e "\n"
# create directory
echo -e "\n>> create save directory"
mkdir -p $SAVE_DIR
echo -e "\n>>create ca certificate"
# 生成ca键
openssl genrsa -aes256 -out $DOCKER_CA_KEY_FILE 4096;
# 生成ca证书
openssl req -subj "/C=CN/ST=Shanghai/L=SH/O=My\ Group/OU=Java\ Development\ Team/CN=$HOST" -new -x509 -days 365 -passin pass:p@ssw0rd -key $DOCKER_CA_KEY_FILE -sha256 -out $DOCKER_CA_CERT_FILE;
echo -e "\n>> create server certificate"
# 生成服务器私钥
openssl genrsa -out $DOCKER_SERVER_KEY_FILE 4096;
# 生成服务端证书配置文件
#echo subjectAltName = DNS:$HOST,IP:192.168.0.60,IP:139.224.247.222 > $CNF_FILE;
echo subjectAltName = DNS:$HOST,IP:$HOST_IP >> $DOCKER_SERVER_CNF_FILE;
echo extendedKeyUsage = serverAuth >> $DOCKER_SERVER_CNF_FILE;
# 生成服务器证书请求
openssl req -subj "/CN=$HOST" -sha256 -new -key $DOCKER_SERVER_KEY_FILE -out $DOCKER_SERVER_CSR_FILE;
#openssl req -subj "/C=CN/ST=Shanghai/L=SH/O=My\ Group/OU=Java\ Development\ Team/CN=$HOST" -sha256 -new -key $DOCKER_SERVER_KEY_FILE -out $DOCKER_SERVER_CSR_FILE;
#openssl req -config $DOCKER_SERVER_CONFIG_FILE -key $DOCKER_SERVER_KEY_FILE -new -sha256 -out $DOCKER_SERVER_CSR_FILE;
# 使用CA证书签发服务器证书
openssl x509 -req -days 365 -sha256 -in $DOCKER_SERVER_CSR_FILE -passin pass:p@ssw0rd -CA $DOCKER_CA_CERT_FILE -CAkey $DOCKER_CA_KEY_FILE -CAcreateserial -out $DOCKER_SERVER_CERT_FILE -extfile $DOCKER_SERVER_CNF_FILE
echo -e "\n>> create client certificate\n"
# 生成客户端私钥
openssl genrsa -out $DOCKER_CLIENT_KEY_FILE 4096
# 生成客户端证书请求
openssl req -subj "/CN=$DOCKER_CLIENT_NAME" -key $DOCKER_CLIENT_KEY_FILE -new -sha256 -out $DOCKER_CLIENT_CSR_FILE;
# 生成客户端证书配置文件
echo extendedKeyUsage = clientAuth >> $DOCKER_CLIENT_CNF_FILE
# 使用CA证书签发客户端证书
openssl x509 -req -days 365 -sha256 -in $DOCKER_CLIENT_CSR_FILE -passin pass:p@ssw0rd -CA $DOCKER_CA_CERT_FILE -CAkey $DOCKER_CA_KEY_FILE -CAcreateserial -out $DOCKER_CLIENT_CERT_FILE -extfile $DOCKER_CLIENT_CNF_FILE
echo -e "\n>> export certificates to p12 format\n"
# 导出证书
echo -e "export $DOCKER_CA_CERT_FILE"
openssl pkcs12 -export -clcerts -in $DOCKER_CA_CERT_FILE -password pass:12123123 -passin pass:p@ssw0rd -inkey $DOCKER_CA_KEY_FILE -out $DOCKER_CA_P12_FILE ;
echo -e "\nexport $DOCKER_SERVER_CERT_FILE"
openssl pkcs12 -export -clcerts -in $DOCKER_SERVER_CERT_FILE -password pass:12123123 -inkey $DOCKER_SERVER_KEY_FILE -out $DOCKER_SERVER_P12_FILE;
echo -e "\nexport $DOCKER_CLIENT_CERT_FILE"
openssl pkcs12 -export -clcerts -in $DOCKER_CLIENT_CERT_FILE -password pass:12123123 -inkey $DOCKER_CLIENT_KEY_FILE -out $DOCKER_CLIENT_P12_FILE;
echo -e "\n>> change privileges of keys and certificates\n"
# 修改证书权限
chmod -v 0400 $DOCKER_CA_KEY_FILE $DOCKER_SERVER_KEY_FILE $DOCKER_CLIENT_KEY_FILe
chmod -v 0444 $DOCKER_CA_CERT_FILE $DOCKER_SERVER_CERT_FILE $DOCKER_CLIENT_CERT_FILE
echo -e "\n 【Finished】"
else
echo "You should provide the domain name!"
fi
2. 编写docker-compose.yml文件
说明：
- 以下文件使用了yaml文件的继承特性，将公有属性提取了出来，放在了
x-minio-common下，并将地址放在了&minio-common
变量中，然后使用*minio-common获取继承的内容，具体请参考docker-compose官方说明
version: '3.7'
# Settings and configurations that are common for all minio containers
x-minio-common: &minio-common
image: minio/minio:$MINIO_VERSION
command: server --address "0.0.0.0:9000" --console-address "0.0.0.0:9001" http://minio{1...4}/data{1...2}
restart: always
expose:
- "9000"
- "9001"
environment:
TZ: UTC
MINIO_ROOT_USER: $MINIO_ROOT_USER
MINIO_ROOT_PASSWORD: $MINIO_ROOT_PASSWORD
extra_hosts:
- $DOCKER_DOMAIN_HOST_NAME:$DOCKER_DOMAIN_HOST_IP
networks:
- minio-net
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
# 定义了4个节点, 每个节点挂载2个卷(类似磁盘)
services:
minio1:
<< : *minio-common
hostname: minio1
volumes:
- data1-1:/data1
- data1-2:/data2
minio2:
<< : *minio-common
hostname: minio2
volumes:
- data2-1:/data1
- data2-2:/data2
minio3:
<< : *minio-common
hostname: minio3
volumes:
- data3-1:/data1
- data3-2:/data2
minio4:
<< : *minio-common
hostname: minio4
volumes:
- data4-1:/data1
- data4-2:/data2
nginx:
image: nginx:1.19.2-alpine
hostname: nginx
volumes:
- ./conf/nginx.conf:/etc/nginx/nginx.conf:ro
- /home/ws/config/ssl/docker/docker-cert.pem:/etc/nginx/certs/public.crt
- /home/ws/config/ssl/docker/docker-key.pem:/etc/nginx/certs/private.key
networks:
- minio-net
ports:
- "49000:9000"
- "49001:9001"
depends_on:
- minio1
- minio2
- minio3
- minio4
## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
## 下面创建挂载卷, 每个挂在卷都是挂在到docker所在主机的磁盘目录
volumes:
data1-1:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data1-1
data1-2:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data1-2
data2-1:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data2-1
data2-2:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data2-2
data3-1:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data3-1
data3-2:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data3-2
data4-1:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data4-1
data4-2:
driver_opts:
type: ext4
o: bind
device: /home/ws/docker/minio/datas/data4-2
# define minionet to create
networks:
minio-net:
name: minionet
driver: bridge
ipam:
driver: default
config:
- subnet: 172.30.6.0/24
gateway: 172.30.6.1
3. 创建环境变量文件 .env
上面的docker-compose文件使用了环境变量，所以需要创建以下的环境变量文件,放在与docker-compose.yml同级的目录下
DOCKER_DOMAIN_HOST_NAME=docker.homeserver.net
DOCKER_DOMAIN_HOST_IP=192.168.3.26
MINIO_VERSION=latest
MINIO_ROOT_USER=AKIAIOSFODNN7EXAPPLE
MINIO_ROOT_PASSWORD=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAPPLEKEY
4. 编写 nginx.conf 文件
把 nginx.conf 文件放在当前minio目录下的conf目录下
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
keepalive_timeout 65;
# enable compress
gzip on;
#自定义变量 $connection_upgrade
map $http_upgrade $connection_upgrade {
default keep-alive; #默认为keep-alive 可以支持 一般http请求
'websocket' upgrade; #如果为websocket 则为 upgrade 可升级的。
}
# include /etc/nginx/conf.d/*.conf;
# minio api loadbalancer
upstream minio {
server minio1:9000;
server minio2:9000;
server minio3:9000;
server minio4:9000;
}
# minio console loadbalancer
upstream console {
ip_hash;
server minio1:9001;
server minio2:9001;
server minio3:9001;
server minio4:9001;
}
server {
listen 9000 ssl;
listen [::]:9000 ssl;
server_name localhost;
# define ssl certificates
ssl_certificate /etc/nginx/certs/public.crt;
ssl_certificate_key /etc/nginx/certs/private.key;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 1000m;
# To disable buffering
proxy_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
proxy_pass http://minio;
}
}
server {
listen 9001 ssl;
listen [::]:9001 ssl;
server_name localhost;
# define ssl certificates
ssl_certificate /etc/nginx/certs/public.crt;
ssl_certificate_key /etc/nginx/certs/private.key;
# To allow special characters in headers
ignore_invalid_headers off;
# Allow any size file to be uploaded.
# Set to a value such as 1000m; to restrict file size to a specific value
client_max_body_size 1000m;
# To disable buffering
proxy_buffering off;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-NginX-Proxy true;
# This is necessary to pass the correct IP to be hashed
real_ip_header X-Real-IP;
proxy_connect_timeout 300;
# To support websocket
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
chunked_transfer_encoding off;
proxy_pass http://console;
}
}
}
5.创建两个脚本文件
create-dir.sh
目的：用于创建目录
#!/bin/bash
mkdir -p ./datas/data1-1 ./datas/data1-2 ./datas/data2-1 ./datas/data2-2 ./datas/data3-1 ./datas/data3-2 ./datas/data4-1 ./datas/data4-2;
operation.sh
目的：启动容器
#!/bin/bash
docker-compose --env-file ./.env -f ./docker-compose.yml $1 $2;
6. 启动MinIO服务器实例
# ./operation.sh up -d
7. 开启防火墙端口
# firewall-cmd --zone=public --add-port=49000/tcp --permanent
# firewall-cmd --zone=public --add-port=49001/tcp --permanent
# firewall-cmd --reload
# firewall-cmd --list-all