개요
○ 기본 구성 요약
openshift-install명령을 사용하여 부팅 가능한 이미지(ISO) 생성 하여 클러스터 배포- 단일 노드 구성(SNO) 3-NODE 클러스터 설치 지원 (이번 가이드는 SNO)
- 완전히 연결되지 않은 배포에서 작동 및 미러링 된 로컬 레지스트리와 함께 작동
○ 최소 사양
| Topology | Master Node | Worker Node | vCPU | Memory | Disk |
| bastion-server | 0 | 0 | 8 vCPU cores | 16GB of RAM | 120GB |
| Single-node cluster | 1 | 0 | 8 vCPU cores | 16GB of RAM | 120GB |
설치 사전 준비
- 부팅 가능한 이미지(ISO)를 만들 Node 1대(RHEL 8.7)로 구성 진행
- 구성 서버는 Bastion 서버로 명명함
Bastion 구성
- localrepo 만들기 – Bastion 서버에서 진행
[root@bastion ~]# mkdir /repository
[root@bastion ~]# mount -t iso9660 -o loop /root/rhel-8.7-x86_64-dvd.iso /repository
[root@bastion ~]# cat <<EOF > /etc/yum.repos.d/local-repo.repo
[BaseOS]
name=RHEL 8 - BaseOS
metadata_expire=-1
gpgcheck=1
enabled=1
baseurl=file:///repository/BaseOS
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
[AppStream]
name=RHEL 8 - AppStream
metadata_expire=-1
gpgcheck=1
enabled=1
baseurl=file:///repository/AppStream/
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
EOF
[root@bastion ~]# yum clean all
[root@bastion ~]# subscription-manager clean
[root@bastion ~]# yum repolist
- SELinux 및 방화벽 종료
[root@bastion ~]# yum update -y
[root@bastion ~]# yum install -y unzip bash-completion httpd bind bind-utils haproxy nfs-utils jq podman
[root@bastion ~]# mkdir -pv /opt/registry/{auth,certs,data}
- 패키지 설치
[root@bastion ~]# yum update -y
[root@bastion ~]# yum install -y unzip bash-completion httpd bind bind-utils haproxy nfs-utils jq podman
[root@bastion ~]# mkdir -pv /opt/registry/{auth,certs,data}
- HTTP 구축
[root@bastion ~]# vim /etc/httpd/conf/httpd.conf
~
45 Listen 8080
~
:wq
[root@bastion ~]# mkdir -pv /var/www/html/ocp
[root@bastion ~]# chmod -R 755 /var/www/html/ocp/
[root@bastion ~]# systemctl enable --now httpd
- NFS 구축
[root@bastion ~]# systemctl enable --now nfs-server
[root@bastion ~]# mkdir -pv /var/nfsshare
[root@bastion ~]# chown -R nobody:nobody /var/nfsshare/ #RHEL7 경우 nfsnobody:nfsnobody
[root@bastion ~]# echo '/var/nfsshare 192.168.0.0/16(rw,sync,root_squash)' >> /etc/exports
[root@bastion ~]# exportfs -r
[root@bastion ~]# systemctl restart nfs-server
- DNS 구축
[root@bastion ~]# vim /etc/named.conf
~
11 listen-on port 53 { any; };
19 allow-query { any; };
~
:wq
- ZONE 추가
[root@bastion ~]# vim /etc/named.rfc1912.zones
```
#문서 하단에 아래 문구 추가
# 정방향
zone "ocp.dhwon.com" IN {
type master;
file "ocp.dhwon.com.zone";
allow-update { none; };
};
# 역방향
zone "1.168.192.in-addr.arpa" IN { # zone 이름이 중요하다. 하단에서 설명한다
type master;
file "ocp.dhwon.com.rev";
allow-update { none; };
};
:wq
# 역방향 적을 시 자신이 사용하는 네트워크 대역 입력 필수
※ 네트워크 대역은 반드시 역방향으로 적을 것
- zone 생성
vi /var/named/ocp.dhwon.com.zone
$TTL 1D
@ IN SOA @ bastion.ocp.dhwon.com. (
0 ; serial
1D ; refresh
1H ; retry
1W ; expire
3H ) ; minimum
; name servers - NS records
NS bastion.ocp.dhwon.com.
; OpenShift Container Platform Cluster - A records
master-1 IN A 192.168.1.87
master-2 IN A 192.168.1.88
master-3 IN A 192.168.1.89
worker-1 IN A 192.168.1.90
worker-2 IN A 192.168.1.91
bootstrap IN A 192.168.1.86
bastion IN A 192.168.1.85
; OpenShift internal cluster IPs - A records
api IN A 192.168.1.85
api-int IN A 192.168.1.85
*.apps IN A 192.168.1.85
- rev 생성
vi /var/named/ocp.dhwon.com.rev
$TTL 1D
@ IN SOA @ bastion.ocp.dhwon.com. (
0 ; serial
1D ; refresh
1H ; retry
1W ; expire
3H ) ; minimum
; name servers - NS records
NS bastion.ocp.dhwon.com.
; OpenShift Container Platform Cluster - PTR records
86 IN PTR bootstrap.ocp.dhwon.com.
87 IN PTR master-1.ocp.dhwon.com.
88 IN PTR master-2.ocp.dhwon.com.
89 IN PTR master-3.ocp.dhwon.com.
90 IN PTR worker-1.ocp.dhwon.com.
91 IN PTR worker-2.ocp.dhwon.com.
85 IN PTR api.ocp.dhwon.com.
85 IN PTR api-int.ocp.dhwon.com.
- zone 체크
[root@bastion ~]# chmod 644 /var/named/ocp.dhwon.com.*
[root@bastion ~]# chmod 644 /var/named/ocp.dhwon.com.*
[root@bastion ~]# named-checkconf /etc/named.conf
[root@bastion ~]# named-checkconf /etc/named.rfc1912.zones
[root@bastion ~]# named-checkzone ocp.dhwon.com /var/named/ocp.dhwon.com.zone
zone ocp.dhwon.com/IN: loaded serial 0
OK
[root@bastion ~]# systemctl enable --now named
[root@bastion ~]# nmcli # dns server에 현재 작업중인 서버의 ip 추가(bastion)
- DNS 정상 작동 확인
[root@bastion ~]# nslookup
# 정방향 조회
> bastion.ocp.dhwon.com
# 역방향 조회
> set type=ptr
> 192.168.1.85
- HAproxy 구축
[root@bastion ~]# mv /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg_bak #원복 백업
[root@bastion ~]# vi /etc/haproxy/haproxy.cfg
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 300s
timeout server 300s
timeout http-keep-alive 10s
timeout check 10s
maxconn 20000
frontend openshift-api-server
bind *:6443
default_backend openshift-api-server
mode tcp
option tcplog
backend openshift-api-server
balance source
mode tcp
server master 192.168.50.52:6443 check fall 3 rise 2
frontend machine-config-server
bind *:22623
default_backend machine-config-server
mode tcp
option tcplog
backend machine-config-server
balance source
mode tcp
server master-1 192.168.50.52:22623 check fall 3 rise 2
frontend ingress-http
bind *:80
default_backend ingress-http
mode tcp
option tcplog
backend ingress-http
balance source
mode tcp
server master 192.168.50.52:80 check fall 3 rise 2
frontend ingress-https
bind *:443
default_backend ingress-https
mode tcp
option tcplog
backend ingress-https
balance source
mode tcp
server master 192.168.50.52:443 check fall 3 rise 2
이미지 미러링
파일 준비
Red Hat OpenShift Container Platform 4.12 for RHEL 8 – Red Hat 고객 포털
[root@bastion ~]# chmod -R 755 /var/www/html/ocp/
[root@bastion ~]# tar zxvf openshift-install-linux-4.10.9.tar
[root@bastion ~]# tar zxvf oc-4.10.9-linux.tar.gz
[root@bastion ~]# mv oc kubectl openshift-install /usr/local/bin/
[root@bastion ~]# openshift-install version
openshift-install 4.10.9
built from commit cf244530e9699eb209036670a17ab471f98f9f9b
release image quay.io/openshift-release-dev/ocp-release@sha256:39f360002b9b5c730d1167879ad6437352d51e72acc9fe80add3ec2a0d20400d
release architecture amd64
[root@bastion ~]# oc version
Client Version: 4.10.9
- Private Image Registry 인증서 적용
[root@bastion ~]# cd /opt/registry/certs
[root@bastion certs]# openssl req -addext "subjectAltName=DNS:bastion.ocp.dhwon.com" -subj "/C=KO/ST=Seoul/L=Seoul/O=Opennaru/OU=support/CN=bastion.ocp.dhwon.com/emailAddress=weondong94@opennaru.com" -newkey rsa:4096 -nodes -sha256 -keyout domain.key -x509 -days 365 -out domain.crt # RHEL7의 경우 RHEL8 서버에서 crt파일 생성 후 copy
[root@bastion certs]# cp domain.crt /etc/pki/ca-trust/source/anchors/.
[root@bastion certs]# update-ca-trust
Pull Secret 파일생성
https://cloud.redhat.com/openshift/install/metal/user-provisioned 에서 Pull secret을 복사합니다.
[root@bastion ~]# vim /root/ocp/pull-secret
`...
...secret 저장...
...`
[root@bastion ocp]# REG_SECRET_INT=`echo -n 'admin:opennaru' | base64 -w0` # harbor 계정 정보를 환경변수에 저장
[root@bastion ocp]# cat /root/ocp/pull-secret | jq '.auths += {"bastion.ocp.dhwon.com:5000": {"auth": "REG_SECRET_INT","email": "weondong94@opennaru.com"}}' | sed "s/REG_SECRET_INT/$REG_SECRET_INT/" > /root/ocp/pull-secret.json # harbor 정보를 추가하여 pull-secret.json 파일 생성
- 이미지 미러링 환경변수 설정
[root@bastion ~]# vim /root/ocp/ocp_env
export OCP_RELEASE=4.12.28 #oc 및 openshift-install 버전과 동일
export LOCAL_REGISTRY='bastion.ocp.dhwon.com:5000'
export LOCAL_REPOSITORY='ocp/ocp4.12.28'
export PRODUCT_REPO='openshift-release-dev'
export LOCAL_SECRET_JSON='/root/ocp/pull-secret.json'
export RELEASE_NAME='ocp-release'
export ARCHITECTURE=x86_64
[root@bastion ~]# source /root/ocp/ocp_env
- 이미지 미러링 실행
[root@bastion ~]# cd /opt/registry/auth/
[root@bastion auth]# htpasswd -cBb ./htpasswd admin opennaru
[root@bastion ~]# podman run --name mirror-registry -p 5000:5000 -v /opt/registry/data:/var/lib/registry:z -v /opt/registry/auth:/auth:z -e "REGISTRY_AUTH=htpasswd" -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -v /opt/registry/certs:/certs:z -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key -e REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED=true -d docker.io/library/registry:2
[root@bastion ~]# podman login -u admin -p opennaru bastion.ocp.dhwon.com:5000
Login Succeeded! # 접속 확인
[root@bastion ~]# export GODEBUG=x509ignoreCN=0; oc adm release mirror -a ${LOCAL_SECRET_JSON} --from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE}-${ARCHITECTURE} --to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} --to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}-${ARCHITECTURE}
- mirrior-registry(podman) systemd 등록
# 재기동 시 container 자동 실행 파일 생성
[root@bastion ~]# podman generate systemd --restart-policy=always -f -t 1 --name mirror-registry
# 복사
[root@bastion ~]# mv container-mirror-registry.service /etc/systemd/system
부팅 할 ISO(CoreOS) 생성하기
# ocp 디렉토리에 설치 진행
[root@bastion ocp]# mkdir -pv /root/ocp/ocp/
- image-config.yaml
[root@bastion ocp]# cd /root/ocp/ocp
cat << EOF > ./install-config.yaml
apiVersion: v1
baseDomain: dhwon.com
metadata:
name: ocp
compute:
- architecture: amd64
hyperthreading: Enabled
name: worker
replicas: 0
controlPlane:
architecture: amd64
hyperthreading: Enabled
name: master
replicas: 1
networking:
clusterNetwork:
- cidr: 10.128.0.0/14
hostPrefix: 23
machineNetwork: # 물리장비 대역대 적어 줄 것
- cidr: 192.168.0.0/16
networkType: OVNKubernetes # SNO 구축시 필수
serviceNetwork:
- 172.30.0.0/16
platform:
none: {}
pullSecret: '{ "auths": { "bastion.ocp.dhwon.com:5000": {"auth": "YWRtaW46b3Blbm5hcnU=","email": "weondong94@opennaru.com" } } }'
sshKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC5UskWOi+j8Gk/rAH80l/8jkEnKx62KU1grvIX2jv4+ykQxtSy/Hp/4yjuSRRkKty6jvhgXagJ7okLkqLJHDbf4fA6/34pSIRaDRio0gIf+YuvkJTFPUn8k5wddqOtzOKJaIijodaNErJvsCTZafWNZ9lynftvs5rZ77NMu2CuhnrblJdDIUzQfrvxBZc+Y1ymtKJnk8aAZTLUbiZcp22E0rbNVgUh0d6/Q0xOO0XMBM+3wquIM/RB7NiZ9lSCEXcAdELDE7x8NEqm0KiCwB+eVb1vfiaA2qUudMa5IGJkgtxDADOLMNRR14nPyK+pwisBF/9hieW9OJq6OpipsQF923Ig5AIYTNB9kCWB6qO0zpScct5scWwLsgImOWkTWJYyLeNl+fiPzdE93ovy3ICcEKriCoJiudHH08emRMwZsE5CaniSnYg6hvNXQO0jNOq7MU5+gVom5e2hF9knahjAIUphJvKczv3ZGQoF85Tq+HhVrH8zgU2usM5FQy2n/53yufJjX7ATBE3xVAmcni5oY8etvJIMh3/uqqQT+1mj+25vgYcGnO7RK5hL5eBOeWY4y5bGbSY7KZBR+XNspsIxtnGptHS9SFGzeCl49E0h7XF+bSd73V9t5AAKSZ5tNMLJ0/LAReZSKXQDwb+apSqVfvtXaGztLLZatoe+ycjNpw== root@bastion.ocp.dhwon.com'
additionalTrustBundle: |
-----BEGIN CERTIFICATE-----
MIIGOTCCBCGgAwIBAgIUYB/WeckzpVrkhfYjKGK1ITVwmCowDQYJKoZIhvcNAQEL
BQAwgZoxCzAJBgNVBAYTAktPMQ4wDAYDVQQIDAVTZW91bDEOMAwGA1UEBwwFU2Vv
dWwxETAPBgNVBAoMCE9wZW5uYXJ1MRAwDgYDVQQLDAdzdXBwb3J0MR4wHAYDVQQD
DBViYXN0aW9uLm9jcC5kaHdvbi5jb20xJjAkBgkqhkiG9w0BCQEWF3dlb25kb25n
OTRAb3Blbm5hcnUuY29tMB4XDTIzMDgxNDAxMjQxNVoXDTI0MDgxMzAxMjQxNVow
gZoxCzAJBgNVBAYTAktPMQ4wDAYDVQQIDAVTZW91bDEOMAwGA1UEBwwFU2VvdWwx
ETAPBgNVBAoMCE9wZW5uYXJ1MRAwDgYDVQQLDAdzdXBwb3J0MR4wHAYDVQQDDBVi
YXN0aW9uLm9jcC5kaHdvbi5jb20xJjAkBgkqhkiG9w0BCQEWF3dlb25kb25nOTRA
b3Blbm5hcnUuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAu2Zn
OHNLbP/pjITBPhYRqeSeKMllfQjXFV968AUu3IGGwqiAhckBhwiOgAN1TwGbhyXe
eIhPPkrVcS0DYC2YML7UUPsyEbJrtlc2cR6RSJkrI/HyUgmCZ6W7P9pjTEegPuHp
fb7XkfRuEa5n28dLul4ZlZgYPT+yvmOdJM+ZnsOO6sxdxmo9YAiHG1UPYL1k2X3x
fWviVFSiwq4phy/PLPzrum2Z3pB+ieorSxYh3E2tlNmpzdPeSw6RxI0J7+6f/3v4
HbQtUev9XR/sWXtThfc8Kd54XJ+1czQ7C79pyjYc3mPpGu0x9rwkR45Ftw+Wd1Vs
+8Bc7zQ2lZNmD1wTH4+rbA7AA2bwrRb6ZEPxXvPNPzSj36ZUtjEwNv1utAzZzVpK
snjdQQBYXQNo83rhmgw1foB1uU+Pv7b8zDz7D7UFRpYL4yEOyx7cIlefcVcbMPgN
mziDzjU84AH4XPvPZFXu77oEiRo//pFWJkM/Hok7r57elUevk67TVK6ag2k+3c9r
vJoyQeAdMqnGY0ZeUJoiGETw8Nf87cwFLPINMGInV9VCu0YmyAYL8ZtvaecyJ53k
yBpZav0DkgUisHHnMz73juhrNkPFzXKyZZ+N9cGkw9VFhiJP6M2iRpBnDAQZ3dEV
8QuoYSQDHaS6SONZLELnygbK4IZy8oLZegRns6sCAwEAAaN1MHMwHQYDVR0OBBYE
FKVEpx0iZ6l1Yjo+7BwkWR6AZGJjMB8GA1UdIwQYMBaAFKVEpx0iZ6l1Yjo+7Bwk
WR6AZGJjMA8GA1UdEwEB/wQFMAMBAf8wIAYDVR0RBBkwF4IVYmFzdGlvbi5vY3Au
ZGh3b24uY29tMA0GCSqGSIb3DQEBCwUAA4ICAQBjt4k08GoeT3meQ6agUOmpydrt
YPktvRvYpI9X1oZ4EGAmNWYQ2X205iSLDZx4v0oHiqf7w3Zj0875Ywr8hPlW0QgQ
6j+h8/8720+NOKFZed3NPHYbryKmtYozHuHosD/rQam7tu2U8LzNbVDWK8DAtEVy
YRi00MU9pQDVlAdfVy0HcWsxRCvfn+IReSfH/0hID6aGFKbiFylmXXwK8wFQSuTa
sjJS94jkEliF94ol2Z/pQJ7bVc5kDmol8AydZN8E8+Q47reRt0IKs8aj8Bj79PzC
Ua9ywtlGkVWjalYppPRyol92b3FX6I6eDLIfkG8xcgvmHPLKOkO3GOZYeoHspIcl
4/CuGPqoULjscgc1bNj923AQ1BuUxtRWpb/yQb/Orl3qygtvBz+vowGU3YU2ffhv
bqPDLMeozEEJibuSuS0ob0ECGxYsp7kx83oPMklL7EK1XTOvkS1vaaO3QF3NRKDS
BECvMw6TshZ4Ve0bRaxdfZZQdcJl4mUx4hdrxzqrZCQ1FszIVkBmnNuKUumc+Xuq
LRaBokR+t1JgklPiNPr+QoUUUXSISErLecvbUH3aRtS+47DnL0LyA+oJjnqe45Ne
L69Vx4T+4fqgxjQPb8FNCCD/hxhFVeaMxfZmlWC5dKge55m+5J0X43VRbGcxj+Dv
OvNPCEuw7X9lZZ87Qw==
-----END CERTIFICATE-----
imageContentSources:
- mirrors:
- bastion.ocp.dhwon.com:5000/ocp/ocp4.12.28
source: quay.io/openshift-release-dev/ocp-release
- mirrors:
- bastion.ocp.dhwon.com:5000/ocp/ocp4.12.28
source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
EOF
- agent-config.yaml 작성
cat << EOF > ./agent-config.yaml
apiVersion: v1alpha1
metadata:
name: ocp
rendezvousIP: 192.168.50.52
hosts:
- hostnam///e: master
interfaces:
- name: ens192
macAddress: 00:50:56:90:9d:b8 #<설치할 가상머신에 MAC Address>
networkConfig:
interfaces:
- name: ens192
type: ethernet
state: up
mac-address: 00:50:56:90:9d:b8 #<설치할 가상머신에 MAC Address>
ipv4:
enabled: true
address:
- ip: 192.168.50.52
prefix-length: 16
dhcp: false
dns-resolver:
config:
server:
- 192.168.50.51
routes:
config:
- destination: 0.0.0.0/0
next-hop-address: 192.168.0.1
next-hop-interface: ens192
table-id: 254
EOF
- 파일 생성 확인
[root@bastion ocp]# ls -l /root/ocp/ocp
total 8
-rw-r--r--. 1 root root 809 Aug 14 01:38 agent-config.yaml
-rw-r--r--. 1 root root 3955 Aug 14 01:38 install-config.yaml
- ISO 파일 생성
[root@bastion ocp]# cd /root/ocp
[root@bastion ocp]# openshift-install agent create image --dir ocp
INFO The rendezvous host IP (node0 IP) is 192.168.50.52
INFO Extracting base ISO from release payload
INFO Verifying cached file
INFO Using cached Base ISO /root/.cache/agent/image_cache/coreos-x86_64.iso
INFO Consuming Install Config from target directory
INFO Consuming Agent Config from target directory
이후 ISO 파일 생성하면 아래와 같은 파일명으로 생성 되어야 함
[root@bastion ocp]# ls -l
total 1173504
-rw-r--r--. 1 root root 1201668096 Aug 14 01:41 agent.x86_64.iso
drwxr-x---. 2 root root 50 Aug 14 01:41 auth
이 후 가상머신을 생성할 수 있는 하이퍼바이저 에 ISO를 넣고 부팅 진행하면 된다.
설치 완료
- 설치 과정 확인
[root@bastion ~] mkdir -pv /root/.kube
[root@bastion ~] cp /root/ocp/ocp/auth/kubeconfig /root/.kube/config
[root@bastion ~] oc get co
- 설치 완료 확인
[root@bastion ~] export KUBECONFIG=/root/ocp/ocp/auth/kubeconfig
[root@bastion ~]# cd /root/ocp/ocp
[root@bastion ocp]# openshift-install agent wait-for install-complet --dir ocp
매주 한 page, 기술 한 spoon 