SplunkK8s/roll_and_resync_buckets_v2.py at main · gjanders/SplunkK8s · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
#run this with splunk cmd python3 or python3
import requests
import json
import time
import sys

wait_for_seconds = 60 * 10
if len(sys.argv) > 1:
    base_url = f"https://{sys.argv[1]}:8089"
    print(f"url to be used is: {base_url}")
else:
    print("No arguments provided, please provide a URL.")
    sys.exit(0)

auth=('roll_buckets_automated', 'roll_buckets_automated_k8s')

url = base_url + "/services/cluster/manager/fixup?output_mode=json&count=0&level=replication_factor"
res = requests.get(url,auth=auth,verify=False)

dict = json.loads(res.text)
print(f"status_code={res.status_code} on url={url}")
#print(dict)

roll_bucket_url = base_url + "/services/cluster/master/control/control/roll-hot-buckets"
resync_bucket_url = base_url + "/services/cluster/master/control/control/resync_bucket_from_peer"

current_time = round(time.time())

for entry in dict['entry']:
    data_latest = entry['content']['latest']
    data_initial = entry['content']['initial']
    if data_latest['reason'].find("bucket hasn't rolled yet") != -1:
        name = entry['name']
        reason = data_latest['reason']
        print(f"bucket={name} requires role due to {reason}")
        bucket_timestamp = data_initial['timestamp']
        diff = current_time - bucket_timestamp
        if diff > wait_for_seconds:
            print(f'bucket={name} requires role due to {reason}, and is beyond {wait_for_seconds} seconds')
            data = { 'bucket_id': name }
            print(f'requests.post("{roll_bucket_url}", data={data}, verify=False)')
            res=requests.post(roll_bucket_url, auth=auth, data=data, verify=False)
            if res.status_code != requests.codes.ok:
                print(f'bucket={name} code={res.status_code} text={res.text}')

        # by this time we have tried to roll the buckets, so now a re-sync might be required instead
        if diff > (wait_for_seconds*2):
            url = base_url + "/services/cluster/master/buckets/" + name + "?output_mode=json"
            res = requests.get(url,auth=auth,verify=False)
            dict_buckets = json.loads(res.text)
            print(f"status_code={res.status_code} on url={url}")
            peer = list(dict_buckets['entry'][0]['content']['peers'].keys())[0]
            data = { 'bucket_id': name, 'peer': peer }
            print(f'requests.post("{resync_bucket_url}", data={data}, verify=False)')
            res=requests.post(resync_bucket_url, auth=auth, data=data, verify=False)
            if res.status_code != requests.codes.ok:
                print(f'bucket={name} code={res.status_code} text={res.text}')

        time.sleep(1)