业务环境介绍

公司当前业务上线流程首先是通过nginx灰度，dubbo-admin操作禁用，然后发布上线主机，发布成功后，dubbo-admin启用，nginx启用主机；之前是通过手动操作，很不方便，本次优化为pipeline方式实现自动发布，需要saltstack api支持。

pipeline发布流程图

准备工作

1. 将saltstack端dubbo.py脚本部署好，可通过salt直接调用

2. 将所有脚本分别放到对应的主机上面（saltstack端、jenkins端）

3. jenkins提前准备好 gitlab、nginx ssh、saltapi 相关凭据，mvn、jdk工具

4. jenkins配置

1. pipeline脚本

// 此pipeline用于nginx + dubbo结构的应用，如没有dubbo，删除dubbo相关步骤即可
// 提前定义：
//     gitlab、nginx ssh、saltapi 相关凭据
//     mvn，jdk等工具定义
//     saltapi模板中servername地址


// 以下所有变量根据实际情况修改
// 本项目的svn代码地址
def svnUrl = 'https://svn****'
// nginx ssh参数，多个以逗号未分隔符
def nginxHosts = '172.87.10.31,172.87.10.41'
// zookeeper主机端口，多个以逗号未分隔符
def zkHostPort = '172.87.40.14:2181,172.87.40.24:2181'
// jenkins发布机脚本绝对路径
def scriptAbsPath = '/app/jenkins_deploy/scripts'
// saltstack 脚本基于salt base路径
def saltScriptPath = 'jenkins_deploy/scripts'
// FTP存储路径映射到saltstack的路径
def ftpSaltPath = 'jenkins_deploy/jenkins_war_packages'
// 项目包名称
def packageName = 'app.war'
// 目包相对路径（相对于$workspace)
def source_file = 'target/' + "${packageName}"
// 应用重启命令
def appRestartCommand = '/app/apache-tomcat-9.0.37/bin/restart.sh app'
// 应用启动用户
def appStartUser = 'app'
// 构建命令
def buildCommand = 'mvn clean package -P product war:war'



// saltapi模板
def Salt(salthost, saltfunc, saltargs) {
    result = salt(authtype: 'pam', 
            clientInterface: local( arguments: saltargs,
                                    function: saltfunc, 
                                    target: salthost, 
                                    targettype: 'list'),
            credentialsId: "saltapi",
            saveFile: true,
            servername: "http://172.87.10.21:8000")

    return result
}


pipeline {
    // 执行任务
    agent any
    // 定义工具
    tools {
        maven 'maven352'
        jdk 'jdk1.8'
    }
    
        
    stages {
        stage("Clone") {
            steps {
                checkout([
                    $class: 'SubversionSCM', 
                    additionalCredentials: [], 
                    excludedCommitMessages: '', 
                    excludedRegions: '', 
                    excludedRevprop: '', 
                    excludedUsers: '', 
                    filterChangelog: false, 
                    ignoreDirPropChanges: false, 
                    includedRegions: '', 
                    locations: [
                        [
                            credentialsId: 'jenkinsprd', 
                            depthOption: 'infinity', 
                            ignoreExternalsOption: true, 
                            local: '.', 
                            remote: "${svnUrl}"
                        ]
                    ], 
                    workspaceUpdater: [$class: 'UpdateUpdater']
                ])
            }
        }
        
        // nginx 禁用upstream主机
        stage("Nginx disable host") {
            steps {
                Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"disabled ${targetHosts} ${nginx_config_file}\"")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"
            }
        }
        
        // nginx 重新加载配置
        stage("Nginx reload 1") {
            input {
                message "是否重启nginx？"
                ok "确定"
                parameters {
                    string(name: "nginx", defaultValue: "${nginx_config_file}", description: '此文件有所改动，请谨慎操作！！！')
                }
            }
            steps {
                Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_reload.sh")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"
            }
        }
        
        // dubbo 禁用生产者
        stage("Dubbo disable") {
            steps {
                script {
                    // 获取主机注册的所有dubbo服务
                    Salt("${targetHosts}","dubbo.ls","${dubbo_port}")
                    dubbo_all_service = sh(script: "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} dubbo.ls", returnStdout:true).trim()
                    // dubbo禁用
                    sh "/usr/bin/python3 ${scriptAbsPath}/dubbo.py disable ${zkHostPort} ${targetHosts} ${dubbo_port} ${dubbo_all_service}"
                }
            }
        }
        
        // mvn构建、代码发布到应用服务器
        stage("mvn build and deploy") {
            steps {
                // 构建命令
                sh "${buildCommand}"
                // 需要修改war包所在的路径
                sh "python3 ${scriptAbsPath}/ftp_upload.py ${WORKSPACE}/${source_file} ${source_file}"
                // /tmp/路径不能修改，这个路径要匹配restart.sh中配置的
                Salt("${targetHosts}","state.sls","""jenkins_deploy.scripts.sync_app_package pillar='{"package_name": "${packageName}"}'""")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} cp.get_file"
                // 项目包权限修改为项目启动用户
                Salt("${targetHosts}","file.chown", "/tmp/${packageName} ${appStartUser} ${appStartUser}")
            }
        }
        
        // 重启应用服务
        stage("restart app service") {
            steps {
                // 应用重启命令
                 Salt("${targetHosts}","cmd.run","'${appRestartCommand}' env=\'{\"LC_ALL\": \"en_US.UTF-8\",\"LANG\": \"en_US.UTF-8\"}\' runas=${appStartUser}")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} cmd.run"
             }
        }
        
        // 测试目标主机是否恢复
        stage("Test") {
            steps {
                println("Test ...")
                sleep 5
            }
        }
        
        // dubbo启用生产者
        stage("Dubbo enable") {
            input {
                message "是否启用dubbo？"
                ok "确定"
                parameters {
                    string(name: "dubbo", defaultValue: "${targetHosts}-${dubbo_port}", description: '请确定应用正常启动，谨慎操作！！！')
                }
            }
            steps {
                script {
                    // 获取主机注册的所有dubbo服务
                    Salt("${targetHosts}","dubbo.ls","${dubbo_port}")
                    dubbo_all_service = sh(script: "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${targetHosts} dubbo.ls", returnStdout:true).trim()
                    // dubbo启用
                    sh "/usr/bin/python3 ${scriptAbsPath}/dubbo.py enable ${zkHostPort} ${targetHosts} ${dubbo_port} ${dubbo_all_service}"
                }
            }
        }
        
        // nginx 启用upstream主机
        stage("Nginx enable host") {
            steps {
                Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"enabled ${targetHosts} ${nginx_config_file}\"")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"
            }
        }
        
        // nginx 重新加载配置
        stage("Nginx reload 2") {
            input {
                message "是否重启nginx？"
                ok "确定"
                parameters {
                    string(name: "nginx", defaultValue: "${nginx_config_file}", description: '此文件有所改动，请谨慎操作！！！')
                }
            }
            steps {
                Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_reload.sh")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"
            }
        }
    }
    
    post {
        success {
            script {
                buildDescription "上次构建成功的主机：${params.targetHosts}"
            }
        }
        
        failure {
            script {
                // nginx 启用upstream主机
                Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"enabled ${targetHosts} ${nginx_config_file}\"")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"
                buildDescription "构建失败，请查看错误！"
            }
        }
        
        aborted {
            script {
                // nginx 启用upstream主机
                Salt("${nginxHosts}","cmd.script","salt://${saltScriptPath}/nginx_upstream.sh \"enabled ${targetHosts} ${nginx_config_file}\"")
                sh "/usr/bin/python3 ${scriptAbsPath}/view_saltoutput.py ${WORKSPACE}/saltOutput.json ${nginxHosts} cmd.script"
                buildDescription "手动取消构建！"
            }
        }
    }
}

2. saltstack server端主机上的脚本

• nginx_reload.sh

#!/bin/bash

source /etc/profile

nginx -t && nginx -s reload && sleep 3 ; ps -ef | grep nginx: | grep -v grep

• nginx_upstream.sh

#!/bin/bash
# Filename   :  nginx_upstream.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  本脚本用于jenkins pipeline发布代码过程中的禁用启用nginx主机操作


SWITCH="$1"
HOSTS="$2"
FILE_NAME="$3"


usage(){
    echo "$0: [enable|enabled|disable|disabled] [hosts] [NGINX_UPSTREAM_CONFIG_FILE]"
}

# 启用nginx主机
enabled(){
    hosts="$1"
    file_name="$2"
    IFS=$','

    for h in ${hosts}
    do
        sed -ri "s/.*($h)/    server \1/g" ${file_name}
    done

    cat ${file_name}
}

# 禁用nginx主机
disabled(){
    hosts="$1"
    file_name="$2"
    IFS=$','

    for h in ${hosts}
    do
        sed -ri "s/.*($h)/#    server \1/g" ${file_name}
    done

    cat ${file_name}
}

# 传递参数不为3，或者任意参数为空则退出脚本
if [ "$#" -ne 3 -o -z "$SWITCH" -o -z "$HOSTS" -o -z "$FILE_NAME" ];then
    usage
    exit 1
fi

case $SWITCH in
    enable|enabled)
        enabled $HOSTS $FILE_NAME 
    ;;
    disable|disabled)
        disabled $HOSTS $FILE_NAME
    ;;
    *)
        usage
        exit 1
    ;;
esac

• sync_app_package.sls

{% set package_name = pillar["package_name"] %}

sync_app_package:
  file.managed:
    - name: /tmp/{{ package_name }}
    - source: salt://jenkins_deploy/jenkins_war_packages/target/{{ package_name }}
    - user: zf
    - group: zf
    - mode: 644

• dubbo.py （此脚本放在saltstack base/_modules目录下面）

# -*- coding: utf-8 -*-
# Filename   :  nginx_upstream.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本用于saltstack自定义模块


from os.path import join as p_join
import telnetlib
import re


__virtualname__ = 'dubbo'
finish = 'dubbo>'


def __virtual__():
    return __virtualname__


def ls(port, host='127.0.0.1', timeout=5):
    '''获取应用注册到dubbo的所有服务的详细信息'''
    tn = telnetlib.Telnet(host=host, port=port, timeout=timeout)
    tn.write('ls -l\n')
    res = tn.read_until(finish).strip(finish)
    tn.close()


    res_l = []
    # 遍历分割成列表的结果
    for service in res.strip('\r\n').split('\r\n'):
        serviceName = re.split('\s+->', service)[0]
        pattern = re.compile('group=(.*?)\&')
        d = pattern.search(service)

        if d:
            res_l.append(p_join(d.group(1), serviceName))
            continue

        res_l.append(serviceName)

    return ','.join(res_l)

• test_url.sh

#!/bin/bash
# Filename   :  test_url.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本用于测试url是否可以访问

URL="$1"
N=0
MAX_N="${2-9999}"

usage(){
    if [ $1 -gt 2 ] || [ $1 -lt 1 ];then
        echo "$0 [URL] [REQUEST_NUMBER]"
        exit 1
    fi
}

Curl(){
    code=$(curl -s -o /dev/null -w '%{http_code}' $URL)

    while true
    do
        if [ "$N" -ge "$MAX_N" ];then
            echo "Test Fail."
            exit 1
        else
            if [ "$code" == 200 ];then
                echo "Test $URL success, Code: $code."
                exit 0
            fi
        fi

        sleep 1
        N=$((N+1))
    done
}

usage $#
Curl

3. jenkins server端主机上面的脚本

• dubbo.py

# -*- coding: utf-8 -*-
# Filename   :  dubbo.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本操作dubbo-admin中服务的启用和禁用

from check_port_connect import check_port
from zookeeper import Zk
from os import path
import urllib.parse
import sys


class Dubbo:
    def __init__(self, zk_host, dubbo_host, dubbo_port, dubbo_all_service, dubbo_path='/dubbo'):
        self.zk_host = zk_host
        self.dubbo_host = dubbo_host
        self.dubbo_port = dubbo_port
        self.dubbo_host_port = [host + ':%s' % self.dubbo_port for host in self.dubbo_host]
        self.dubbo_path = dubbo_path
        self.zk = Zk(self.zk_host)
        self.all_node = dubbo_all_service
        # 写入zookeeper对应dubbo的服务配置模板
        self.template = 'override://%s/%s?category=configurators&disabled=true&dynamic=false&enabled=true'
        self._check_port()
 

    def _check_port(self):
        # 检测dubbo主机的端口连通性
        for host in self.dubbo_host:
            dubbo_port_test = check_port(host, self.dubbo_port)
            if dubbo_port_test != 0: 
                sys.exit(1) 

    def _format(self, zk_node, dubbo_host_port):
        # 分割服务名，group/service 根号分割的名称是有组的服务
        l = zk_node.split('/')

        # 大于1就是有组的服务
        if len(l) > 1:
            zk_node = l[1]
            override = self.template % (dubbo_host_port, l[1])
            override += '&group=' + l[0]
        else:
            zk_node = l[0]
            override = self.template % (dubbo_host_port, l[0])

        return zk_node, override

    def disable(self):
        # 循环所有node并禁用所有dubbo主机的服务
        for zk_node in self.all_node:
            for host_port in self.dubbo_host_port:
                zk_node, override = self._format(zk_node, host_port)
                d = urllib.parse.urlencode({'name': override}).split('name=')[1]
                self.zk.create('%s/%s/configurators/%s' % (self.dubbo_path, zk_node, d), b'[]')

        print('dubbo %s的服务已禁用，请查看WEB页面！' % ','.join(self.dubbo_host_port))
        self.zk.stop()

    def enable(self):
        # 循环所有node并禁用所有dubbo主机的服务
        for zk_node in self.all_node:
            for host_port in self.dubbo_host_port:
                zk_node, override = self._format(zk_node, host_port)
                d = urllib.parse.urlencode({'name': override}).split('name=')[1]
                self.zk.delete('%s/%s/configurators/%s' % (self.dubbo_path, zk_node, d))

        print('dubbo %s的服务已启用，请查看WEB页面！' % ','.join(self.dubbo_host_port))
        self.zk.stop()

    def servic_status(self):
        for host_port in self.dubbo_host_port:
            n = 0
            for zk_node in self.all_node:
                zk_node, override = self._format(zk_node, host_port)
                d = urllib.parse.urlencode({'name': override}).split('name=')[1]
                res = self.zk.ls('%s/%s/configurators' % (self.dubbo_path, zk_node))

                # 如果格式化后的值存在于列表中，即为禁用
                if d not in res:
                    n += 1
        
            print('Host: %s, Total: %s, Enable: %s' % (host_port, len(self.all_node), n))


if __name__ == '__main__':
    usage = 'usgae: %s [enable|disable|service_status] [zk_host:port,...] [dubbo_host] [dubbo_port] [dubbo_all_service]' % sys.argv[0]
    if len(sys.argv) != 6:
        sys.exit(usage)

    method = sys.argv[1] 
    zk_host = sys.argv[2]
    dubbo_host = sys.argv[3].split(',')
    dubbo_port = sys.argv[4]
    dubbo_all_service = sys.argv[5].split(',')
    dubbo = Dubbo(zk_host=zk_host, dubbo_host=dubbo_host, dubbo_port=dubbo_port, dubbo_all_service=dubbo_all_service)

    if method == 'disable':
        dubbo.disable()
    elif method == 'enable':
        dubbo.enable()
    elif method == 'service_status':
        dubbo.servic_status()
    else:
        sys.exit(usage)

• zookeeper.py

# -*- coding: utf-8 -*-
# Filename   :  zookeeper.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本用于连接、操作zookeeper


from kazoo.client import KazooClient
from kazoo.exceptions import NoNodeError
from check_port_connect import check_port
from sys import exit


class Zk:
    def __init__(self, hosts, timeout=10):
        self.hosts = hosts
        self.timeout = timeout
        self._zk = KazooClient(hosts=self.hosts, timeout=self.timeout)
        self._zk.start()

    def ls(self, path='/'):
        '''
        获取Zk执行path中所有node
        :return: 所有node
        '''
        try:
            all_node = self._zk.get_children(path)
            return all_node
        except Exception as e:
            print(repr(e))
            exit(1) 

    def create(self, path, data):
        '''
        Zk创建
        :return:
        '''
        try:
            res = self._zk.create(path, data, makepath=True)
            return res
        except Exception as e:
            print(repr(e))
            exit(1)

    def delete(self, path):
        '''
        Zk 删除节点
        :param path: 要删除的节点
        :return:
        '''
        try:
            res = self._zk.delete(path)
            return res
        except Exception as e:
            return repr(e)

    def stop(self):
        self._zk.stop()

• view_saltoutput.py

# -*- coding: utf-8 -*-
# Filename   :  nginx_upstream.sh
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本用于查看saltstack api的输出结果，因为saltapi中定义了saveFile: true


import json, sys


def outPutJson(file_path, hosts, module):
    '''
    解析saltapi执行结果输出的json文件
    切记，不同模块输出格式有所不同
    '''
    with open(file_path, 'r') as read_f:
        data = json.load(read_f)
        for host in hosts:
            try:
                if module == 'cmd.script':
                    res = (host, data[0][host]['ret']['stdout'])
                elif module == 'cmd.run':
                    res = (host + '(stdout)', data[0][host]['ret'])
                elif module == 'cp.get_file':
                    res = (host, data[0][host])
                    if not res[1]:
                        sys.exit(res)
                elif module == 'dubbo.ls':
                    res = data[0][host]
                    print(res)
                    break
                else:
                    res = ''
                print('%s {\n%s\n}' % res)
            except KeyError as e:
                print('KeyError: %s' % e)
                sys.exit(1)


if __name__ == '__main__':
    usage = "usage: %s [SALT_OUTPUT_FILE] [HOSTS] [MODULE]" % sys.argv[0]

    if len(sys.argv) != 4:
        sys.exit(usage) 

    file_path = sys.argv[1]
    hosts = sys.argv[2].split(',')
    module = sys.argv[3]
    outPutJson(file_path, hosts, module)

• ftp_upload.py

# -*- coding: utf-8 -*-
# Filename   :  ftp_upload.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本用于操作ftp

from ftplib import FTP
from os import path
import sys


class FTPClient:
    def __init__(self, host, user, passwd, port=21):
        self.host = host
        self.user = user
        self.passwd = passwd
        self.port = port
        self._ftp = None
        self._login()

    def _login(self):
        '''
        登录FTP服务器
        :return: 连接或登录出现异常时返回错误信息
        '''
        try:
            self._ftp = FTP()
            self._ftp.connect(self.host, self.port, timeout=30)
            self._ftp.login(self.user, self.passwd)
        except Exception as e:
            print(str(e))
            sys.exit(1)

    def upload(self, localpath, remotepath=None):
        '''
        上传ftp文件
        :param localpath: local file path
        :param remotepath: remote file path
        :return:
        '''
        if not localpath: return 'Please select a local file. '
        # 读取本地文件
        fp = open(localpath, 'rb')

        # 如果未传递远程文件路径，则上传到当前目录，文件名称同本地文件
        if not remotepath:
            remotepath = path.basename(localpath)

        # 上传文件
        try:
            self._ftp.storbinary('STOR ' + remotepath, fp)
        except Exception as e:
            print(str(e))

    def nlst(self, dir='/'):
        '''
        查看目录下的内容
        :return: 以列表形式返回目录下的所有内容
        '''
        files_list = self._ftp.nlst(dir)
        return files_list

    def del_file(self, filename=None):
        '''
        删除文件
        :param filename: 文件名称
        :return: 执行结果
        '''
        if not filename: return 'Please input filename'

        try:
            del_f = self._ftp.delete(filename)
        except Exception as e:
            print(str(e))
            sys.exit(1)

    def close(self):
        '''
        退出ftp连接
        :return:
        '''
        try:
            # 向服务器发送quit命令
            self._ftp.quit()
        except Exception:
            print('No response from server')
            sys.exit(1)
        finally:
            # 客户端单方面关闭连接
            self._ftp.close()


if __name__ == '__main__':
    usage = "usage: %s [FILE_NAME] [TARGET_PATH]" % sys.argv[0]
    ftp_host = '172.85.10.31'
    ftp_user = 'jenkins_deploy'
    ftp_password = 'jenkins@deploy_2021'

    if len(sys.argv) != 3:
        sys.exit(usage)

    ftp = FTPClient(host=ftp_host, user=ftp_user, passwd=ftp_password)
    file_path = sys.argv[1]
    target_path = sys.argv[2]
    all_file = ftp.nlst('target')

    if file_path.startswith('/'):
        file_path = path.join('/', file_path)

    f_path = 'target/' + path.basename(file_path)
    if f_path in all_file: 
        ftp.del_file(f_path)

    ftp.upload(file_path, target_path)
    ftp.close()

• check_port_connect.py

# -*- coding: utf-8 -*-
# Filename   :  check_port_connect.py
# Date       :  2021/08/26
# Author     :  beiguohao   
# Email      :  oct_hao@163.com
# Description:  此脚本用于检测端口连通性


import socket
import sys


socket.setdefaulttimeout(10) 

def check_port(ip, port):
    port = int(port)
    sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    res = sk.connect_ex((ip, port))
    
    if res == 0:
        # print('%s %s port is Open' % (ip, port))
        return 0
    else:
        print('%s %s port is Not Open' % (ip, port))
        sys.exit(1) 

 
if __name__ == '__main__':
    usage = 'usage: %s [IP|HSOTNAME] [PORT]'
    if len(sys.argv) != 3:
        sys.exit(usage) 

    ip = sys.argv[1]    
    port = sys.argv[2]
    check_port(ip, port)