春风十里不如你 —— Taozi - 监控 https://xiongan.host/index.php/tag/%E7%9B%91%E6%8E%A7/ 基于Kubernetes集群的监控网络服务 https://xiongan.host/index.php/archives/226/ 2023-10-30T01:41:00+08:00 基于Kubernetes集群的监控网络服务介绍需要以下环境Kubernetes集群Blackbox工具Grafana、Prometheus监控大致功能:通过在K8s集群中部署blackbox工具(用于监控服务,检查网络可用性)和Grafana、Prometheus(监控可视化面板)更直观的体现网络连通性,可以进行警报和分析本文章通过若海博客的【Kubernetes 集群上安装 Blackbox 监控网站状态】和【Kubernetes 集群上安装 Grafana 和 Prometheus】整合而成部署Kubernetes集群(Ubuntu/Debian操作系统)确保主节点和子节点都有Docker环境(最好是同一个版本)主节点//安装Docker,一键安装(如有安装可以忽略) curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun //开启docker、并设置开机自启 systemctl start docker & systemctl enable docker apt update apt install -y wireguard echo "net.ipv4.ip_forward = 1" >/etc/sysctl.d/ip_forward.conf sysctl -p /etc/sysctl.d/ip_forward.conf //以下Token值请保存,任意字符串 export SERVER_TOKEN=r83nui54eg8wihyiteshuo3o43gbf7u9er63o43gbf7uitujg8wihyitr6 export PUBLIC_IP=$(curl -Ls http://metadata.tencentyun.com/latest/meta-data/public-ipv4) export PRIVATE_IP=$(curl -Ls http://metadata.tencentyun.com/latest/meta-data/local-ipv4) export INSTALL_K3S_SKIP_DOWNLOAD=true export DOWNLOAD_K3S_BIN_URL=https://github.com/k3s-io/k3s/releases/download/v1.28.2%2Bk3s1/k3s if [ $(curl -Ls http://ipip.rehi.org/country_code) == "CN" ]; then   DOWNLOAD_K3S_BIN_URL=https://ghproxy.com/${DOWNLOAD_K3S_BIN_URL} fi curl -Lo /usr/local/bin/k3s $DOWNLOAD_K3S_BIN_URL chmod a+x /usr/local/bin/k3s curl -Ls https://get.k3s.io | sh -s - server \   --cluster-init \   --token $SERVER_TOKEN \   --node-ip $PRIVATE_IP \   --node-external-ip $PUBLIC_IP \   --advertise-address $PRIVATE_IP \   --service-node-port-range 5432-9876 \   --flannel-backend wireguard-native \   --flannel-external-ip子节点//安装Docker,一键安装(如有安装可以忽略) curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun //开启docker、并设置开机自启 systemctl start docker & systemctl enable docker //子节点代码 apt update apt install -y wireguard echo "net.ipv4.ip_forward = 1" >/etc/sysctl.d/ip_forward.conf sysctl -p /etc/sysctl.d/ip_forward.conf export SERVER_IP=43.129.195.33 //此ip填你的主节点地址 export SERVER_TOKEN=r83nui54eg8wihyiteshuo3o43gbf7u9er63o43gbf7uitujg8wihyitr6 export PUBLIC_IP=$(curl -Ls http://metadata.tencentyun.com/latest/meta-data/public-ipv4) export PRIVATE_IP=$(curl -Ls http://metadata.tencentyun.com/latest/meta-data/local-ipv4) export INSTALL_K3S_SKIP_DOWNLOAD=true export DOWNLOAD_K3S_BIN_URL=https://github.com/k3s-io/k3s/releases/download/v1.28.2%2Bk3s1/k3s if [ $(curl -Ls http://ipip.rehi.org/country_code) == "CN" ]; then   DOWNLOAD_K3S_BIN_URL=https://ghproxy.com/${DOWNLOAD_K3S_BIN_URL} fi curl -Lo /usr/local/bin/k3s $DOWNLOAD_K3S_BIN_URL chmod a+x /usr/local/bin/k3s curl -Ls https://get.k3s.io | sh -s - agent \   --server https://$SERVER_IP:6443 \   --token $SERVER_TOKEN \   --node-ip $PRIVATE_IP \   --node-external-ip $PUBLIC_IPBlackbox工具部署(也有集群方式)//拉取镜像 docker pull rehiy/blackbox //一键启动 docker run -d \   --name blackbox \   --restart always \   --publish 9115:9115 \   --env "NODE_NAME=guangzhou-taozi" \   --env "NODE_OWNER=Taozi" \   --env "NODE_REGION=广州" \   --env "NODE_ISP=TencentCloud" \   --env "NODE_BANNER=From Taozii-www.xiongan.host" \   rehiy/blackbox //开始注册 docker logs -f blackboxGrafana、Prometheus部署在主节点创建一个目录,名字任意,然后在同一目录中创建两个文件(grafpro.yaml、grafpro.sh)grafpro.yamlkind: Deployment apiVersion: apps/v1 metadata: name: &name grafpro labels:   app: *name spec: selector:   matchLabels:     app: *name template:   metadata:     labels:       app: *name   spec:     initContainers:       - name: busybox         image: busybox         command:           - sh           - -c           - |             if [ ! -f /etc/prometheus/prometheus.yml ]; then             cat <<EOF >/etc/prometheus/prometheus.yml             global:               scrape_timeout: 25s               scrape_interval: 1m               evaluation_interval: 1m             scrape_configs:               - job_name: prometheus                 static_configs:                   - targets:                       - 127.0.0.1:9090             EOF             fi         volumeMounts:           - name: *name             subPath: etc             mountPath: /etc/prometheus     containers:       - name: grafana         image: grafana/grafana         securityContext:           runAsUser: 0         ports:           - containerPort: 3000         volumeMounts:           - name: *name             subPath: grafana             mountPath: /var/lib/grafana       - name: prometheus         image: prom/prometheus         securityContext:           runAsUser: 0         ports:           - containerPort: 9090         volumeMounts:           - name: *name             subPath: etc             mountPath: /etc/prometheus           - name: *name             subPath: prometheus             mountPath: /prometheus     volumes:       - name: *name         hostPath:           path: /srv/grafpro           type: DirectoryOrCreate --- kind: Service apiVersion: v1 metadata: name: &name grafpro labels:   app: *name spec: selector:   app: *name ports:   - name: grafana     port: 3000     targetPort: 3000   - name: prometheus     port: 9090     targetPort: 9090 --- kind: Ingress apiVersion: networking.k8s.io/v1 metadata: name: &name grafpro annotations:   traefik.ingress.kubernetes.io/router.entrypoints: web,websecure spec: rules:   - host: grafana.example.org     http:       paths:         - path: /           pathType: Prefix           backend:             service:               name: *name               port:                 name: grafana   - host: prometheus.example.org     http:       paths:         - path: /           pathType: Prefix           backend:             service:               name: *name               port:                 name: prometheus tls:   - secretName: defaultgrafpro.sh//警告:请修改路径和访问域名 # 配置存储路径 export GRAFPRO_STORAGE=${GRAFPRO_STORAGE:-"/srv/grafpro"} # 配置访问域名 export GRAFANA_DOMAIN=${GRAFPRO_DOMAIN:-"grafana.example.org"} export PROMETHEUS_DOMAIN=${PROMETHEUS_DOMAIN:-"prometheus.example.org"} # 修改参数并部署服务 cat grafpro.yaml \   | sed "s#/srv/grafpro#$GRAFPRO_STORAGE#g" \   | sed "s#grafana.example.org#$GRAFANA_DOMAIN#g" \   | sed "s#prometheus.example.org#$PROMETHEUS_DOMAIN#g" \   | kubectl apply -f -部署chmod +x grafpro.sh ./grafpro.sh测试打开注意以下,开启端口9115、9090 浏览器打开地址http://grafana.example.org 账号密码都是admin,首次登录,提示修改密码,修改后自动跳到控制台 浏览器打开http://grafana.example.org/connections/datasources/选择第一个,然后编辑URL为:http://127.0.0.1:9090 然后保存 然后选择创建好的Prometheus,导入面板 浏览器打开http://prometheus.example.org,查看信息配置Promethues任务//回到主节点的/srv/grafpro/etc目录下 编辑yml文件,备份一下原有的yml,创建新的yml mv prometheus.yml prometheus00.yml //以下是yml文件内容(若部署时修改了负载名称blackbox-exporter,下文的配置文件也要做相应的修改) global: scrape_timeout: 15s scrape_interval: 1m evaluation_interval: 1m scrape_configs: # prometheus - job_name: prometheus   static_configs:     - targets:         - 127.0.0.1:9090 # blackbox_all - job_name: blackbox_all   static_configs:     - targets:         - blackbox-gz:9115       labels:         region: '广州,腾讯云' # http_status_gz - job_name: http_status_gz   metrics_path: /probe   params:     module: [http_2xx] #配置get请求检测   static_configs:     - targets:         - https://www.example.com       labels:         project: 测试1         desc: 测试网站描述1     - targets:         - https://www.example.org       labels:         project: 测试2         desc: 测试网站描述2   basic_auth:     username: ******     password: ******         relabel_configs:     - target_label: region       replacement: '广州,腾讯云'     - source_labels: [__address__]       target_label: __param_target     - source_labels: [__param_target]       target_label: instance     - target_label: __address__       replacement: blackbox-gz:9115:80然后重启svc,方法如下:首先查看podkubectl get pod 然后删除查看到关于grafana的pod,然后稍等几分钟即可 kubectl delete pod *导入 Grafana 仪表盘下载附件json在Grafana仪表盘里导入即可导入后可以查看到监控仪已经开始了,显示各项信息 【Docker】k8s健康检查 https://xiongan.host/index.php/archives/212/ 2023-05-28T12:05:24+08:00 健康检查使用存活探针创建使用 execaction 模式的存活探针 pod 的 yaml 文件。需要创建目录(/tmp/healthy)查看到运行成功,持续监控pod状态,看到pod反复重启使用 describe 命令查看详细 pod 信息,正常创建使用 http 存活探针的 pod 的 yaml 文件。创建yaml运行并查看状态查看详细events创建使用 tcp 存活探针的 pod 的 yaml,模板采用 httpd 容器镜像。创建yaml文件运行并进行容器内操作查看pod的restarts次数查看pod之前未通过liveness的记录就绪探针创建 http 的 deployment 的 yaml 文件,其中配置 readiness 探针。运行deployment使用describechakanhttp服务的endpoint可以看到有4个地址进入一个容器,删除index.html文件再使用 describe 命令查看 endpoint可以看到删除的pod地址已经从endpoint中移除查看pod的详细信息,看到pod未通过探针检测查看pod信息,kandaopod处于notready状态 【Zabbix】部署监控软件 https://xiongan.host/index.php/archives/190/ 2022-12-15T17:43:00+08:00 介绍zabbix是一个监控软件,其可以监控各种网络参数,保证企业服务架构安全运营,同时支持灵活的告警机制,可以使得运维人员快速定位故障、解决问题。zabbix支持分布式功能,支持复杂架构下的监控解决方案,也支持web页面,为主机监控提供了良好直观的展现。部署安装httpd和php7服务端:[root@srv-tz ~]# yum install -y[root@srv-tz ~]# systemctl enable --now httpd客户端:[root@client01 ~]# yum install -y yum-plugin-priorities && yum install http://rpms.famillecollet.com/enterprise/remi-release-7.rpm -y #修改repo配置文件 [root@client01 ~]# sed -i -e "s/\]$/\]\npriority=10/g" /etc/yum.repos.d/remi-safe.repo [root@client01 ~]# sed -i -e "s/enabled=1/enabled=0/g" /etc/yum.repos.d/remi-safe.repo #安装php扩展 [root@client01 ~]# yum --enablerepo=remi-safe,epel install php72 php72-php-pear php72-php-mbstring -y #启动和自启php [root@client01 ~]# scl enable php72 bash #查看php版本信息 [root@client01 ~]# php -v PHP 7.2.34 (cli) (built: Oct 24 2022 10:27:24) ( NTS ) Copyright (c) 1997-2018 The PHP Group Zend Engine v3.2.0, Copyright (c) 1998-2018 Zend Technologies编辑脚本文件 [root@client01 ~]# vim /etc/profile.d/php72.sh #!/bin/bash source /opt/remi/php72/enable export X_SCLS="`scl enable php72 'echo $X_SCLS'`" #安装php从remi源中 [root@client01 ~]# yum --enablerepo=remi-safe,epel -y install php72-php [root@client01 ~]# systemctl enable --now httpd #写入页面 [root@client01 ~]# echo '<?php phpinfo(); ?>' > /var/www/html/info.php #查看页面 [root@client01 ~]# curl http://localhost/info.php | grep 'PHP Version' | tail -1 | sed --e 's/<[^>]*>//g' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 69230 0 69230 0 0 6822k 0 --:--:-- --:--:-- --:--:-- 7511k PHP Version 7.2.34安装及配置 MariaDB安装环境服务端:[root@srv-tz ~]# yum install centos-release-scl-rh centos-release-scl -y #修改配置文件 [root@srv-tz ~]# sed -i -e "s/\]$/\]\npriority=10/g" /etc/yum.repos.d/CentOS-SCLo-scl.repo [root@srv-tz ~]# sed -i -e "s/\]$/\]\npriority=10/g" /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo [root@srv-tz ~]# sed -i -e "s/enabled=1/enabled=0/g" /etc/yum.repos.d/CentOS-SCLo-scl.repo [root@srv-tz ~]# sed -i -e "s/enabled=1/enabled=0/g" /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo #安装 [root@srv-tz ~]# yum --enablerepo=centos-sclo-rh install rh-mariadb103-mariadb-server -y启用 MariaDB 环境#运行mariadb [root@srv-tz ~]# scl enable rh-mariadb103 bash #查看版本 [root@srv-tz ~]# mysql -V mysql Ver 15.1 Distrib 10.3.35-MariaDB, for Linux (x86_64) using EditLine wrapper #写脚本 [root@srv-tz ~]# vim /etc/profile.d/rh-mariadb103.sh #!/bin/bash source /opt/rh/rh-mariadb103/enable export X_SCLS="`scl enable rh-mariadb103 'echo $X_SCLS'`" #启动运行 [root@srv-tz my.cnf.d]# systemctl enable --now rh-mariadb103-mariadb #开始部署安装 [root@srv-tz my.cnf.d]# mysql_secure_installation 开始需要设置一个密码 按照提示进行确认即可最后会提示安装成功 All done! If you've completed all of the above steps, your MariaDB installation should now be secure. Thanks for using MariaDB!安装 Zabbix Server[root@srv-tz ~]# yum install https://repo.zabbix.com/zabbix/5.0/rhel/7/x86_64/zabbix-release-5.0-1.el7.noarch.rpm -y [root@srv-tz ~]# yum-config-manager --enable zabbix-frontend [root@srv-tz ~]# yum --enablerepo=centos-sclo-rh install zabbix-server-mysql zabbix-web-mysql-scl zabbix-apache-conf-scl zabbix-agent zabbix-get -y配置 Zabbix Server配置 Zabbix Server 数据库#登录数据库 [root@srv-tz ~]# mysql -uroot -p123456 MariaDB [(none)]> create database zabbix character set utf8 collate utf8_bin; Query OK, 1 row affected (0.001 sec) MariaDB [(none)]> grant all privileges on zabbix.* to zabbix@'localhost' identified by 'password'; Query OK, 0 rows affected (0.028 sec) MariaDB [(none)]> flush privileges; Query OK, 0 rows affected (0.001 sec) [root@srv-tz ~]# cd /usr/share/doc/zabbix-server-mysql-5.0.30/ [root@srv-tz zabbix-server-mysql-5.0.30]# ls AUTHORS ChangeLog COPYING create.sql.gz double.sql NEWS README [root@srv-tz zabbix-server-mysql-5.0.30]# gunzip create.sql.gz [root@srv-tz zabbix-server-mysql-5.0.30]# mysql -u root -p zabbix < create.sql 输入密码123456设置SElinux[root@srv-tz ~]# setsebool -P zabbix_can_network on [root@srv-tz ~]# setsebool -P httpd_can_connect_zabbix on [root@srv-tz ~]# setsebool -P domain_can_mmap_files on [root@srv-tz ~]# setsebool -P daemons_enable_cluster_mode on [root@srv-tz ~]# vim zabbix_server.te module zabbix_server 1.0; require { type zabbix_t; type zabbix_agent_t; type rpm_exec_t; type rpm_var_lib_t; class file { execute execute_no_trans map open }; class capability dac_override; } #============= zabbix_t ============== allow zabbix_t self:capability dac_override; #============= zabbix_agent_t ============== allow zabbix_agent_t rpm_var_lib_t:file open; allow zabbix_agent_t rpm_exec_t:file { execute execute_no_trans map }; [root@srv-tz ~]# checkmodule -m -M -o zabbix_server.mod zabbix_server.te checkmodule: loading policy configuration from zabbix_server.te checkmodule: policy configuration loaded checkmodule: writing binary representation (version 19) to zabbix_server.mod [root@srv-tz ~]# semodule_package --outfile zabbix_server.pp --module zabbix_server.mod [root@srv-tz ~]# semodule -i zabbix_server.ppFirewall设置[root@srv-tz ~]# firewall-cmd --add-service={http,https} --permanent success [root@srv-tz ~]# firewall-cmd --add-port={10050/tcp,10051/tcp} --permanent success [root@srv-tz ~]# firewall-cmd --reload success配置 Zabbix Agentd[root@srv-tz ~]# vim /etc/zabbix/zabbix_agentd.conf //* 更改 117 行,指定 Zabbix Server 的 IP Server=127.0.0.1 //* 更改 158 行,指定 Zabbix Server 的 IP ServerActive=127.0.0.1 //* 更改 169 行,指定 Zabbix Server 的 FQDN Hostname=srv-tz [root@srv-tz ~]# systemctl enable --now zabbix-agent为 Zabbix Server 配置 httpd 服务[root@srv-tz ~]# vim /etc/httpd/conf.d/zabbix.conf //* 更改 10 行,允许指定网络访问 #Require all granted Require ip 127.0.0.1 192.168.123.0/24 #定义 zabbix 的 timezone [root@srv-tz ~]# vim /etc/opt/rh/rh-php72/php-fpm.d/zabbix.conf //* 更改 24 行 php_value[date.timezone] = Asia/Shanghai [root@srv-tz ~]# systemctl enable --now httpd rh-php72-php-fpm访问webhttp:ip/zabbix进行页面安装配置数据库 账号zabbix 密码为password安装成功后默认账号Admin 密码zabbix