k8s部署MinIO Operator

k8s部署MinIO Operator"/>

k8s部署MinIO Operator

Minio Operator

环境

[root@master-10 minio]# kubectl get no -o wide
NAME        STATUS   ROLES    AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION           CONTAINER-RUNTIME
master-10   Ready    master   104d    v1.19.3   192.168.91.10   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-11     Ready    node     44m     v1.19.3   192.168.91.11   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-12     Ready    node     104d    v1.19.3   192.168.91.12   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-13     Ready    node     3d16h   v1.19.3   192.168.91.13   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
node-14     Ready    node     3d16h   v1.19.3   192.168.91.14   <none>        CentOS Linux 7 (Core)   3.10.0-1160.el7.x86_64   docker://19.3.15
[root@master-10 minio]# kubectl top no 
NAME        CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master-10   183m         2%     2148Mi          27%       
node-11     74m          1%     807Mi           22%       
node-12     76m          1%     876Mi           23%       
node-13     86m          2%     874Mi           23%       
node-14     88m          2%     779Mi           21% 

前期准备

创建目录

#db001
mkdir -p /home/data/local-volume/minio/v1/node-pv0
#master1
mkdir -p /home/data/local-volume/minio/v1/node-pv1
#node001
mkdir -p /home/data/local-volume/minio/v1/node-pv2
#node002
mkdir -p /home/data/local-volume/minio/v1/node-pv3

创建PV和SC

#v1-node-pv-sc.yaml
#静态存储
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-node-v1-pv0
spec:capacity:storage: 10Gi  volumeMode: FilesystemaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: node-local-storagelocal:path: /home/data/local-volume/minio/v1/node-pv0 nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- db001
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-node-v1-pv1
spec:capacity:storage: 10GivolumeMode: FilesystemaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: node-local-storagelocal:path: /home/data/local-volume/minio/v1/node-pv1nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- master1---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-node-v1-pv2
spec:capacity:storage: 10GivolumeMode: FilesystemaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: node-local-storagelocal:path: /home/data/local-volume/minio/v1/node-pv2nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- node001
---
apiVersion: v1
kind: PersistentVolume
metadata:name: minio-node-v1-pv3
spec:capacity:storage: 10GivolumeMode: FilesystemaccessModes:- ReadWriteOncepersistentVolumeReclaimPolicy: RetainstorageClassName: node-local-storagelocal:path: /home/data/local-volume/minio/v1/node-pv3 nodeAffinity:required:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/hostnameoperator: Invalues:- node002
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: node-local-storage
provisioner: kubernetes.io/no-provisioner
volumeBindingMode: WaitForFirstConsumer

创建namespace

[root@master1 minio]# kubectl create namespace minio-node
namespace/minio-node created

检查

[root@master1 minio]# kubectl get sc
NAME                 PROVISIONER                    RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
node-local-storage   kubernetes.io/no-provisioner   Delete          WaitForFirstConsumer   false                  40s[root@master1 minio]# kubectl get pv,ns -A | grep minio
persistentvolume/minio-node-v1-pv0   10Gi       RWO            Retain           Available                         node-local-storage            10s
persistentvolume/minio-node-v1-pv1   10Gi       RWO            Retain           Available                         node-local-storage            10s
persistentvolume/minio-node-v1-pv2   10Gi       RWO            Retain           Available                         node-local-storage            10s
persistentvolume/minio-node-v1-pv3   10Gi       RWO            Retain           Available                         node-local-storage            10s
namespace/minio-node                Active   26s
namespace/minio-operator            Active   4d23h

安装MinIO Operator

kubecrew

1. Make sure that git is installed.

2. Run this command to download and install krew:

   项目地址： .4.1/krew.tar.gz
   wget .4.1/krew.yaml
   tar -zxvf krew.tar.gz
   ./krew-linux_amd64 install --manifest=krew.yaml --archive=krew.tar.gz
   

3. Add the $HOME/.krew/bin directory to your PATH environment variable. To do this, update your .bashrc or .zshrc file and append the following line:

   export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"
   echo 'export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"' >> ~/.bashrc
   source ~/.bashrc
   

   and restart your shell.

4. Run kubectl krew to check the installation.

   kubectl krew
   

#krew会依赖git，注意git是否正确安装且写入环境变量
[root@master-10 minio]# kubectl krew update
Updated the local copy of plugin index.
#被墙容易失败,多试几次
[root@master-10 minio]# kubectl krew install minio
F0310 11:03:25.508814   73405 root.go:58] failed to update the local index: fetch index at "/root/.krew/index" failed: command execution failure, output="fatal: unable to access '.git/': TCP connection reset by peer\n": exit status 128
[root@master-10 minio]# kubectl krew install minio
Updated the local copy of plugin index.
Installing plugin: minio
Installed plugin: minio
\| Use this plugin:| 	kubectl minio| Documentation:| 	| Caveats:| \|  | * For resources that are not in default namespace, currently you must|  |   specify -n/--namespace explicitly (the current namespace setting is not|  |   yet used).| /
/
WARNING: You installed a plugin from the krew-index plugin repository.These plugins are not audited for security by the Krew maintainers.Run them at your own risk.#######离线

wget .4.11/kubectl-minio_4.4.11_linux_amd64 -O kubectl-minio
chmod +x kubectl-minio
mv kubectl-minio /usr/local/bin/#Run the following command to verify installation of the plugin:
[root@master-10 minio]# kubectl minio version
v4.4.11#Run the following command to initialize the Operator:
[root@master-10 minio]# kubectl minio init --cluster-domain=cluster.saas
namespace/minio-operator created
serviceaccount/minio-operator created
clusterrole.rbac.authorization.k8s.io/minio-operator-role created
clusterrolebinding.rbac.authorization.k8s.io/minio-operator-binding created
customresourcedefinition.apiextensions.k8s.io/tenants.minio.min.io created
service/operator created
deployment.apps/minio-operator created
serviceaccount/console-sa created
clusterrole.rbac.authorization.k8s.io/console-sa-role created
clusterrolebinding.rbac.authorization.k8s.io/console-sa-binding created
configmap/console-env created
service/console created
deployment.apps/console created
-----------------To open Operator UI, start a port forward using this command:kubectl minio proxy -n minio-operator 
-----------------
[root@master-10 ~]# kubectl get pods -n minio-operator
NAME                              READY   STATUS    RESTARTS   AGE
console-58b7dc8856-xlcbf          1/1     Running   2          22h
minio-operator-849f8cf8b8-ldq9x   1/1     Running   2          21h
minio-operator-849f8cf8b8-wf4qw   1/1     Running   2          22h

进入控制台

#新建终端
[root@master-10 minio]# kubectl minio proxy -n minio-operator 
Starting port forward of the Console UI.To connect open a browser and go to http://localhost:9090Current JWT to login: eyJhbGciOiJSUzI1NiIsImtpZCI6InY4NkNBSHptSUZhbXIyS2VWMk4yc3Rua2M0bUQyc2tYZVVHVVVDWEE1Y3cifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtaW5pby1vcGVyYXRvciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJjb25zb2xlLXNhLXRva2VuLTkyODU3Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImNvbnNvbGUtc2EiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI4OGI1MDhjMS01YmI0LTQ0MDAtOWIyZi05YTMxYzZmZWJiYmMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bWluaW8tb3BlcmF0b3I6Y29uc29sZS1zYSJ9.s0ycCfkyvGmhxKJULjTz56yipt0T_viP5GTiTfTS3yBawYpiL-d9usO-Ot9H-7gRTKT9euykENeQF3D4lhpSeVlATD0RjFwY4D-Ks7MT3v2UWJ5j5GItpY6qMHsS-cKuRVosSRhgyFf9Fc4He_CbY8e5GdbWTW-0P138FmEPcrogrTw37AjHySjKbXrLzGEL84rdGTL2lgRpmPwQWXYerchNaGNFO5ir3PJGf7PYkmKfDByq02P_yhT4U70JgLhNgPwT9i0MzHnSORUFKyuNRaIoSW6-tgh7Wk2eEUTZI79QgIsaJ6GH4-oDNZcSf6mHKNZbEkaUt6VgoBR0VYZhJgForwarding from 0.0.0.0:9090 -> 9090#复制JWT进入控制台
http://192.168.91.10:9090/

创建租户

minio-operator

minio-node
UfrFvdv9Wturm5FO
A6UNtGUr3QESHLzkSTwqdaNUNiyNRpBt

命令行方式

[root@master-10 minio]# kubectl minio tenant create tenant1 --namespace minio-tenant-1 --storage-class local-storage --servers 4 --volumes 4 --capacity 40GiTenant 'tenant1' created in 'minio-tenant-1' NamespaceUsername: admin Password: 933c9ff5-3ff5-4331-b95c-477830cf8825 Note: Copy the credentials to a secure location. MinIO will not display these again.+-------------+-----------------+----------------+--------------+--------------+
| APPLICATION | SERVICE NAME    | NAMESPACE      | SERVICE TYPE | SERVICE PORT |
+-------------+-----------------+----------------+--------------+--------------+
| MinIO       | minio           | minio-tenant-1 | ClusterIP    | 443          |
| Console     | tenant1-console | minio-tenant-1 | ClusterIP    | 9443         |
+-------------+-----------------+----------------+--------------+--------------+

检查

[root@master1 minio]# kubectl get all -n minio-node 
NAME                      READY   STATUS    RESTARTS   AGE
pod/minio-node-pool-0-0   1/1     Running   0          57s
pod/minio-node-pool-0-1   1/1     Running   0          56s
pod/minio-node-pool-0-2   1/1     Running   0          56s
pod/minio-node-pool-0-3   1/1     Running   0          56sNAME                         TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
service/minio                LoadBalancer   10.233.31.36   <pending>     80:31819/TCP     59s
service/minio-node-console   LoadBalancer   10.233.38.9    <pending>     9090:30786/TCP   58s
service/minio-node-hl        ClusterIP      None           <none>        9000/TCP         57sNAME                                 READY   AGE
statefulset.apps/minio-node-pool-0   4/4     57s#######
The minio service corresponds to the MinIO Tenant service. Applications should use this service for performing operations against the MinIO Tenant.
The minio-tenant-1-console service corresponds to the MinIO Console. Administrators should use this service for accessing the MinIO Console and performing administrative operations on the MinIO Tenant.
The minio-tenant-1-hl corresponds to a headless service used to facilitate communication between Pods in the Tenant. 

删掉重建

[root@master1 minio]# kubectl delete pvc --all -n minio-node
[root@master1 minio]# kubectl delete -f v1-node-pv-sc.yaml [root@master1 minio]# kubectl delete secrets -n minio-node  minio-node-secret
secret "minio-node-secret" deleted

#新
lmfmRBFopIQS0Pmn
ZP823zRd7cz85rkcHfhyEQDIaq2OOAav

创建ingress

openssl genrsa -out tls.key 2048#创建密钥    域名为www.minio
openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Guangdong/L=Guangzhou/O=devops/CN=www.minio
#创建secret
kubectl create secret tls minio-secret --cert=tls.crt --key=tls.key  -n minio-node[root@master1 minio]# kubectl create secret tls minio-secret --cert=tls.crt --key=tls.key  -n minio-node
secret/minio-secret created

ingress.yaml

[root@master1 minio]# cat minio-node-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:annotations:nginx.ingress.kubernetes.io/ssl-redirect: "true"nginx.ingress.kubernetes.io/use-regex: "true"name: minio-ingressnamespace: minio-node
spec:
#  tls:
#  - hosts: 
#    - www.minio
#    secretName: minio-secretrules:- host: www.miniohttp:paths:- path: /backend:serviceName: minioservicePort: 80

创建ingress

[root@master1 minio]# kubectl create -f minio-node-ingress.yaml 
Warning: extensions/v1beta1 Ingress is deprecated in v1.14+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
ingress.extensions/minio-ingress created

访问tenant

在windows端设置域名解析

192.168.0.84 www.minio

浏览器访问

Python SDK

Download using pip

pip3 install minio

Download source

git clone .git
cd minio-py
python3 setup.py install  #linux

file_uploader.py

from minio import Minio
from minio.error import S3Errordef main():# Create a client with the MinIO server playground, its access key# and secret key.client = Minio("www.minio",access_key="UfrFvdv9Wturm5FO",secret_key="A6UNtGUr3QESHLzkSTwqdaNUNiyNRpBt",secure="false",)# Make 'test1' bucket if not exist.found = client.bucket_exists("test1")if not found:client.make_bucket("test1")else:print("Bucket 'test1' already exists")# Upload '/root/kube-yaml/minio/v1-node-pv-sc.yaml' as object name# 'v1-node-pv-sc.yaml' to bucket 'test1'.client.fput_object("test1", "v1-node-pv-sc.yaml", "/root/kube-yaml/minio/v1-node-pv-sc.yaml",)print("'/root/kube-yaml/minio/v1-node-pv-sc.yaml' is successfully uploaded as ""object 'v1-node-pv-sc.yaml' to bucket 'test1'.")if __name__ == "__main__":try:main()except S3Error as exc:print("error occurred.", exc)

Run File Uploader

$ python3 file_uploader.py
'/home/user/Photos/asiaphotos.zip' is successfully uploaded as object 'asiaphotos-2015.zip' to bucket 'asiatrip'.$ mc ls play/test1/
[2016-06-02 18:10:29 PDT]  82KiB asiaphotos-2015.zip