bdfiles/main.py at main · iportman/bdfiles · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""
    Batch download files main
"""

import requests
import yaml
import os
import time


def download(save_dir=None, is_test=False):
    # load config file
    conf_dict = get_conf(is_test)

    # get config info
    if save_dir is None:
        save_dir = conf_dict['save_dir']

    download_url_path = conf_dict['download_url_path']
    download_files = conf_dict['download_files']

    # create dir
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    # download files
    for file_name in download_files:
        download_url = download_url_path + file_name
        print('download url: ' + download_url)

        r = requests.get(download_url, verify=False)
        with open(save_dir + file_name, 'wb') as f:
            f.write(r.content)

        print(file_name + ' download success.')

    print('all files download success!')

    if not is_test:
        print('program exit after 5 seconds...')
        time.sleep(5)


def get_conf(is_test=False):
    conf_file_name = os.sep + 'conf.yml'
    if is_test:
        conf_file_name = os.sep + '..' + conf_file_name

    conf_file_path = os.getcwd() + conf_file_name
    if not os.path.exists(conf_file_path):
        raise Exception('config file does not exists.')

    conf_file = open(conf_file_path, 'r')
    conf_data = conf_file.read()
    conf_file.close()

    return yaml.safe_load(conf_data)


if __name__ == "__main__":
    download()