pgcsvlog_parser/csvfile_common.py at master · i0seph/pgcsvlog_parser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# coding:utf-8
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import sys, csv
from io import StringIO

def tail_f(fn, nSeek):
    f = open(fn, 'r', encoding='latin1', newline='')
    f.seek(nSeek)
    startseek = nSeek;
    try:
        tempstr = StringIO(f.read());
    except UnicodeDecodeError as e:
        sys.stderr.write('** decode error' + str(e) + '\n')
        sys.stderr.flush()
        yield(startseek, [])
        return;
    r = csv.reader(tempstr)
    for arr in r:
        if len(arr) == 23:
            startseek = tempstr.tell() + nSeek;
            yield (startseek, arr)
        else:
            yield (startseek, arr)
            break;
    tempstr.close()
    del tempstr
    f.close()

def makedict(arr, where):
    aDicKey = [
        'log_time',
        'user_name',
        'database_name',
        'process_id',
        'connection_from',
        'session_id',
        'session_line_num',
        'command_tag',
        'session_start_time',
        'virtual_transaction_id',
        'transaction_id',
        'error_severity',
        'sql_state_code',
        'message',
        'detail',
        'hint',
        'internal_query',
        'internal_query_pos',
        'context',
        'query',
        'query_pos',
        'location',
        'application_name'];

    textcols = (
        'user_name',
        'database_name',
        'message',
        'detail',
        'hint',
        'internal_query',
        'context',
        'query',
    );

    aDict = {}
    val = None;
    for i in range(len(aDicKey)):
        if arr[i] == '': continue
        try:
            dummy  = int(arr[i])
            if dummy == 0: continue
        except ValueError as e:
            pass
        if aDicKey[i] in textcols:
            val = arr[i].encode('latin1').decode('utf8');
        else:
            val = arr[i]
        if aDicKey[i] == 'message' and val.find("duration: ") == 0:
            aDict['duration'] = str(int(round(float(val[10:10 + val[10:].find(' ')]))))
            if val.find(' ms  statement: ') > 0:
                aDict['query'] = val[val.find('statement: ') + 11:]
            elif val.find(' ms  plan:\n') > 0:
                aDict['plan'] = val[val.find('\n') + 1:]
        else:
            aDict[aDicKey[i]] = val
    del val
    return (aDict, where);