问题描述
我们有一个简单的管道，可以将我们的 API 从注册集群推广到我们的生产集群。
我们正在添加新集群，当我将新集群添加到我的管道时，我得到了一些奇怪的行为。
当新集群是最后一个被处理的时候它会在调用部署后挂起，所以我们前两个集群的代码会部署并返回成功响应，然后在第三个集群开始部署，响应一半然后更改就挂了，如果我登录到我们的第三个集群，它将显示它已成功部署 API。
在某些情况下，我取得了成功，如果 pod 缩放为零，它就会起作用。 如果新集群是它唯一需要部署到的集群，它也可以工作。 最后，我改变了集群的顺序，所以新集群现在是第一个要处理的集群，这很有效。
有没有人知道我能看到什么可能是造成这种情况的原因？
为了部署和监控，我只是使用带有状态的选择器来运行部署本身。
def rm = openshift.selector("dc","${APP_NAME}").rollout()
rm.latest()
rm.status()
它挂在 rm.status() 上，所以它会通过 3 次调用，然后不会再做任何事情。
这是完整的管道（我编辑了一些敏感代码）
// vars/ocp_prod_promotion.groovy(Destination Project,"Project" name)
def call(body) {
def pipelineParams = [:]
body.resolveStrategy = Closure.DELEGATE_FirsT
body.delegate = pipelineParams
body()
pipeline {
options {
disableConcurrentBuilds()
}
agent {
node {
label 'openshift'
}
}
environment{
//<<redacted,if its all caps below it came from up here>>
}
stages{
stage("Determine Apps to promote"){
steps{
script{
params = []
openshift.withCluster("${SRC_CLUSTER}"){
openshift.withProject("${SRC_PROJECT}"){
// get all images with project label (extract app name and image digest)
def all_istag = openshift.selector("istag",[domain: "${pipelineParams.DOMAIN_NAME}"]) // need to label the istag (make image promotion a lot cleaner/nicer)
echo "build list of registry apps from (${SRC_PROJECT}) with domain (${pipelineParams.DOMAIN_NAME})"
def reg_list = []
all_istag.withEach {
def istag = openshift.selector("${it.name()}").object()
reg_list.add(istag)
}
echo "determine if single app or multiple app environment"
if(reg_list.size()==1) {
echo "single app found"
def appKey = "${reg_list[0].Metadata.name.split(":")[0]}"
def shaKey = "${reg_list[0].image.Metadata.name}"
INPUT_ParaMS = ["${appKey}":true]
timeout(time: 20,unit: 'MINUTES') {
input message: "Ready to promote ${appKey}(${shaKey}) to Production?"
}
} else {
echo "multiple apps found"
reg_list.each { regImage ->
// boolean parameter - description is image digest value,name is the app name (no tag added)
params.add(booleanParam(defaultValue: false,description: regImage.image.Metadata.name,name: regImage.Metadata.name.split(":")[0]))
}
timeout(time: 20,unit: 'MINUTES') {
INPUT_ParaMS = input message: 'Please Select Apps to Promote',ok: 'Next',parameters: params
}
}
}
}
}
}
}
stage ("Get secrets") {
steps {
script {
echo "Backup configmaps"
sh "mkdir -p /tmp/prodconfig && cp -R ./* /tmp/prodconfig"
checkout([$class : 'GitSCM',branches : [[name: "*/master"]],userRemoteConfigs: [[url: "${GIT_SECRETS_URL}",credentialsId: "GitLabCreds"]]
]);
sh "mkdir -p prodconfig && cp -R /tmp/prodconfig/* ./prodconfig"
}
}
}
stage ("Prod promotion") {
steps{
script{
sh '''
docker login -u $SRC_CREDS_USR -p $SRC_CREDS_PSW https://$SRC_REGISTRY
docker login -u $DEST_1B_CREDS_USR -p $DEST_1B_CREDS_PSW https://$DEST_1B_REGISTRY
docker login -u $DEST_2B_CREDS_USR -p $DEST_2B_CREDS_PSW https://$DEST_2B_REGISTRY
docker login -u $DEST_1A_CREDS_USR -p $DEST_1A_CREDS_PSW https://$DEST_1A_REGISTRY
'''
def pod_counts = ''
echo "Check if there is a pod_count.json file in the stage config"
if (fileExists('./pod_count.json')) {
echo "pod_count.json found!"
pod_counts = readJSON file: './pod_count.json'
} else {
echo "No pod_count.json found - using default pod count for all promotions"
}
// loop through all input params
INPUT_ParaMS.each { item ->
appName = item.key
selected = item.value
if (selected == true) {
echo "Generate and promote template for ${appName}"
def dest_replica_count = '2'
// Check if pod_counts has value
echo "check pod_counts value and use this value for pods to be spun up"
if (pod_counts) {
// Use the app name to get the value for pod count
pod_count = pod_counts["${appName}"]
// check if pod_count has value
if (pod_count) {
dest_replica_count = pod_count
echo "Pod count for ${appName}: ${dest_replica_count}"
} else {
echo "Using default pod count for ${appName}: ${dest_replica_count}"
}
} else {
echo "Using default pod count for ${appName}: ${dest_replica_count}"
}
// call function to do the promotions
echo "promote ${appName} to all clusters"
prod_promote(appName,DEST_1A_CLUSTER,DEST_1A_CREDS_PSW,DEST_1A_REGISTRY,dest_replica_count)
prod_promote(appName,DEST_1B_CLUSTER,DEST_1B_CREDS_PSW,DEST_1B_REGISTRY,DEST_2B_CLUSTER,DEST_2B_CREDS_PSW,DEST_2B_REGISTRY,dest_replica_count)
}
}
sh '''
docker logout https://$DEST_1B_REGISTRY
docker logout https://$DEST_2B_REGISTRY
docker logout https://$DEST_1A_REGISTRY
docker logout https://$SRC_REGISTRY
'''
}
}
}
}
post {
always {
// clear out the docker config file,which may cause issues but still better to do
sh "echo '' > /var/lib/jenkins/.docker/config.json"
// Clean up tmp prodconfig
dir("/tmp/prodconfig"){
deleteDir()
}
}
}
}
}
void prod_promote(APP_NAME,CLUSTER,CLUSTER_CREDS,DESTINATION,REPLICA_COUNT) {
// generate template
openshift.withCluster("${CLUSTER}","${CLUSTER_CREDS}"){
openshift.withProject("${DEST_PROJECT}"){
// Deployment already exists,so don't trigger it
if (openshift.selector("dc","${APP_NAME}").exists()) {
openshift.set("triggers","dc/${APP_NAME}","--manual")
}
// push image to production cluster - with docker / docker config stuff sorted and inside openshift.withCluster
image_promo_string = "${SRC_REGISTRY}/${SRC_PROJECT}/${APP_NAME}:${SRC_TAG} ${DESTINATION}/${DEST_PROJECT}/${APP_NAME}:prod"
openshift.raw("image mirror","${image_promo_string}")
}
}
// do the template Now
openshift.withCluster("${SRC_CLUSTER}"){
openshift.withProject("${SRC_PROJECT}"){
if (!openshift.selector("template","${APP_NAME}").exists()) {
echo "${APP_NAME} does not exist in ${SRC_PROJECT},cannot deploy"
return
}
// get template
template = openshift.selector("template","${APP_NAME}").object( exportable:true )
// get image labels
img = openshift.selector("is","${APP_NAME}").object()
labels = img.Metadata.labels
echo "${APP_NAME} -- ${labels}"
}
}
// promote template and apply it to production
openshift.withCluster("${CLUSTER}","${CLUSTER_CREDS}"){
openshift.withProject("${DEST_PROJECT}"){
// update image labels first
img = openshift.selector("is","${APP_NAME}").object()
img['Metadata']['labels'] = labels
openshift.apply(img)
// create and modify the template
if (openshift.selector("template","${APP_NAME}").exists()) {
openshift.replace(template)
} else {
openshift.create(template)
}
def t = openshift.selector('template',"${APP_NAME}").object()
for (obj in t.objects) {
if (obj.spec) {
if (obj.spec.template) {
obj.spec.replicas = "2".toInteger()
obj.spec.template.spec.containers[0].image = ''
// vital ! let the trigger fill this in
for (trig in obj.spec.triggers) {
if (trig.imageChangeParams) {
// vital ! let the trigger fill this in
trig.imageChangeParams.from.name = "${APP_NAME}:prod"
trig.imageChangeParams.from.namespace = "${DEST_PROJECT}"
trig.imageChangeParams.automatic = false
}
}
}
}
}
// store template with new params (note image trigger false)
openshift.replace(t)
// create new objects
//<<redacted environment variables>>
def rm = openshift.selector("dc","${APP_NAME}").rollout()
rm.latest()
rm.status()
openshift.set("triggers","--auto")
// scale pods
openshift.selector("dc","${APP_NAME}").scale("--replicas=${REPLICA_COUNT}")
openshift.selector("dc","${APP_NAME}").rollout().status()
}
}
}
解决方法
暂无找到可以解决该程序问题的有效方法，小编努力寻找整理中！
如果你已经找到好的解决方法，欢迎将解决方案带上本链接一起发送给小编。
小编邮箱:dio#foxmail.com (将#修改为@）