Big data, hadoop installation and deployment
** Kubernetes **
sudo /home/sealer/sealer2/_output/bin/sealer/linux_amd64/sealer run docker.io/sealerio/kubernetes:v1.22.15 --masters 10.0.0.245 --nodes 10.0.0.246,10.0.0.247 --user sealer --passwd ********
Hadoopdocker hub
docker pull kubeguide/hadoop:latest
** hadoop.yaml **
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-hadoop-conf
namespace: default
data:
HDFS_MASTER_SERVICE: hadoop-hdfs-master
HDOOP_YARN_MASTER: hadoop-yarn-master
---
apiVersion: v1
kind: Service
metadata:
name: hadoop-hdfs-master
spec:
type: NodePort
selector:
app: hdfs-master
ports:
- name: rpc
port: 9000
targetPort: 9000
- name: http
port: 50070
targetPort: 50070
nodePort: 32007
---
apiVersion: v1
kind: Pod
metadata:
name: hdfs-master
labels:
app: hdfs-master
spec:
containers:
- name: hdfs-master
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9000
- containerPort: 50070
env:
- name: HADOOP_NODE_TYPE
value: namenode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Pod
metadata:
name: hadoop-datanode-1
labels:
app: hadoop-datanode-1
spec:
containers:
- name: hadoop-datanode-1
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9000
- containerPort: 50070
env:
- name: HADOOP_NODE_TYPE
value: datanode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Pod
metadata:
name: hadoop-datanode-2
labels:
app: hadoop-datanode-2
spec:
containers:
- name: hadoop-datanode-2
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9000
- containerPort: 50070
env:
- name: HADOOP_NODE_TYPE
value: datanode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Pod
metadata:
name: hadoop-datanode-3
labels:
app: hadoop-datanode-3
spec:
containers:
- name: hadoop-datanode-3
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9000
- containerPort: 50070
env:
- name: HADOOP_NODE_TYPE
value: datanode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: hadoop-yarn-master
spec:
type: NodePort
selector:
app: yarn-master
ports:
- name: "8030"
port: 8030
- name: "8031"
port: 8031
- name: "8032"
port: 8032
- name: http
port: 8088
targetPort: 8088
nodePort: 32088
---
apiVersion: v1
kind: Pod
metadata:
name: yarn-master
labels:
app: yarn-master
spec:
containers:
- name: yarn-master
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9000
- containerPort: 50070
env:
- name: HADOOP_NODE_TYPE
value: resourceman
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: yarn-node-1
spec:
clusterIP: None
selector:
app: yarn-node-1
ports:
- port: 8040
---
apiVersion: v1
kind: Service
metadata:
name: yarn-node-2
spec:
clusterIP: None
selector:
app: yarn-node-2
ports:
- port: 8040
---
apiVersion: v1
kind: Service
metadata:
name: yarn-node-3
spec:
clusterIP: None
selector:
app: yarn-node-3
ports:
- port: 8040
---
apiVersion: v1
kind: Pod
metadata:
name: yarn-node-1
labels:
app: yarn-node-1
spec:
containers:
- name: yarn-node-1
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8040
- containerPort: 8041
- containerPort: 8042
env:
- name: HADOOP_NODE_TYPE
value: yarnnode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Pod
metadata:
name: yarn-node-2
labels:
app: yarn-node-2
spec:
containers:
- name: yarn-node-2
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8040
- containerPort: 8041
- containerPort: 8042
env:
- name: HADOOP_NODE_TYPE
value: yarnnode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
---
apiVersion: v1
kind: Pod
metadata:
name: yarn-node-3
labels:
app: yarn-node-3
spec:
containers:
- name: yarn-node-3
image: 192.168.242.132/library/kubernetes-hadoop:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8040
- containerPort: 8041
- containerPort: 8042
env:
- name: HADOOP_NODE_TYPE
value: yarnnode
- name: HDFS_MASTER_SERVICE
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDFS_MASTER_SERVICE
- name: HDOOP_YARN_MASTER
valueFrom:
configMapKeyRef:
name: kube-hadoop-conf
key: HDOOP_YARN_MASTER
restartPolicy: Always
yamlConfigMap5Service8podConfigMapHDFS_MASTER_SERVICEHDOOP_YARN_MASTERIPHDFS servicedatanodenamenodeipc.Client: Retrying connect to server: xxx:9000.
kubectl create -f hadoop.yaml
2022627 18:49:58
: C:\Users\smile\Desktop\git\big-date
Mode LastWriteTime Length Name
---- ------------- ------ ----
d----- 2022/6/27 18:48 markdowm
-a---- 2022/6/27 18:45 4957395 hadooop.docx
-a---- 2022/6/27 18:45 3870121 hadoop.zip
-a---- 2022/4/8 15:58 3275 HADOOP.md
-a---- 2022/6/27 18:49 56 README.md
-a---- 2022/4/4 21:42 70002 solution.md
-a---- 2022/6/27 18:45 0 TOC.md
-a---- 2022/6/27 18:45 2279 .md
-a---- 2022/6/27 18:45 43376 .md