对象存储过期回收巡检通知

‍

一、背景

由于业务使用云厂商对象存储，时间过久会存在历史桶，未回收，几经波折 交接，无人认领，对后续运维和安全带来较大风险和挑战，通过云厂拉取的桶监控信息存储到promethues。
如下，可用prom脚本巡检。查询进一个月请求数为0的桶，通知相关同学回收

1、巡检脚本

cat loop_checkloop_check_ks3.py

#-*- coding: UTF-8 -*- 
#python3
import openpyxl ,json
import requests  # 导入requests模块

webhook_test = 'https://sd.a.cn/api/v1/webhook/send?key=hsjshjksdhfdjksgksgds'
webhook_yunchang = 'https://sd.a.cn/api/v1/webhook/send?key=dhakjhdadhakshdaskhdajkhdajks'
search_prom_url = "http://vm-promthues.a.cn/select/0/prometheus/api/v1/query"

team_file = './team.json'

def get_prometheus_data(url, query):   # 定义函数，用于从Prometheus获取监控数据
    params = {'query': query}          # 设置请求参数，包含要查���的metric信息
    r = requests.post(url, params=params)  # 根据URL和参数进行GET请求

    if r.status_code == 200:            # 判断是否成功获得响应
        return r.json()                 # 返回JSON格式的响应内容。
    else:                               # 处理错误情况。
        print(r.text)
        print('Error occurred while getting data from Prometheus.')  

def get_res(data_list):
    alert_dict = {}
    for item in data_list:
        ks3_name = item['metric']['bucket']
        # project_id = item['metric'].get('project_id', 'None') if 'project_id' in item['metric'] else 'None'
        team_id = item['metric'].get('team_id','None') if 'team_id' in  item['metric']  else 'None'

        alert_dict[ks3_name] = {'account': item['metric']['account'],'reqtotal': item['value'][1], 'team_id': team_id,'cloud_provider': item['metric']['cloud_provider'] }

    return alert_dict

def get_projectid_to_team(teamid):
    with open(team_file, "r", encoding='utf-8') as f:
        content = f.read()
        jsondata = json.loads(content) 
    f.close() 
    team_list = []
    team_list = jsondata['data']
    for team in team_list:
        if team['kae_project_id'] == teamid:
             return team['name'], team['responsible_person_name']

def render_msg(alertmsg):
    msg = {
        "msgtype": "markdown",
        "markdown": {
            "text": f"***对象存储桶近一月0访问巡检*** nn"
            f"相关桶情况: nn"
            f"桶名称 / 服务提供商 / 主账号 / 团队负责人 nn"
            f"{alertmsg} "
        }
    }
    return msg   

def send_webhook(msg):
    # req=requests.post(webhook_test,data=json.dumps(msg))
    #webhook_yunchang
    req=requests.post(webhook_yunchang,data=json.dumps(msg))
    if req.status_code == 200:
        return req.json()
    else:
        "go back to response"

if __name__ == "__main__":

    ks3_reqtotal_query = 'sum_over_time(obs_req_count{}[30d])'
    respone_ks3req = get_prometheus_data(search_prom_url,ks3_reqtotal_query)
    if respone_ks3req['status'] == 'success':
        res_list = respone_ks3req['data']['result'] 
        print("result :"+str(len(res_list)) )
    else:
        print(respone_ks3req['status'])
        sys.exit(1)

    ks3req_alert_dict = get_res(res_list)
    alert_msg = ''
    for k ,v in ks3req_alert_dict.items():
        if v['reqtotal'] == '0':
            if v['team_id'] != 'None':
                team_name = get_projectid_to_team(v['team_id'])
            else:
                team_name = 'NoTeam'
            # print(k,v['cloud_provider'],v['account'],team_name)
            alert_msg = alert_msg + k +' '+ v['cloud_provider']+' '+v['account']+' '+str(team_name) +"nn"
    sendmsg = render_msg(alert_msg)
    send_webhook(sendmsg)

2、展示结果