data-pipeline/import_cicero_curl_broken.py at master · enactdev/data-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
#!/usr/bin/python

import os

import pycurl
from StringIO import StringIO

import json

from urllib import urlencode

from models import session, PoliticalDonation, CiceroDistrict, PoliticalDonationContributor, PoliticalDonationContributorAddress
from models import PoliticalDonationContributorAddressCiceroDetails, PoliticalDonationContributorAddressCiceroRaw


token = '4kz-a90df93759f3875046dc'
user = ''# 1346

# Function to do our API calls (returns object made from JSON)
# Copied from Cicero's PHP example
# http://cicero.azavea.com/docs/hello_world_php.html
def get_cicero_response(url, postfields=''):

    #print('postfields:', postfields)

    buffer = StringIO()
    c = pycurl.Curl()
    c.setopt(c.URL, url)
    c.setopt(c.WRITEDATA, buffer)

    if postfields != '':
        #print('adding post fields option')
        c.setopt(c.POSTFIELDS, postfields)

    c.perform()
    c.close()

    result_json = buffer.getvalue()
    # Body is a string in some encoding.
    # In Python 2, we can print it without knowing what the encoding is.
    #print(result_json)

    #print result_json

    #print dir(result_json)

    return json.loads(result_json)


def get_cicero_legislatrive_response(address, token, user):


    # Check for address from addr1 and zipcode5
    try:

        existing_response = session.query(PoliticalDonationContributorAddressCiceroRaw)\
            .filter(PoliticalDonationContributorAddressCiceroRaw.addr1==address.addr1)\
            .filter(PoliticalDonationContributorAddressCiceroRaw.zipcode5==address.zipcode[:5])\
            .one()

        print 'returning cached response'

        return json.loads(existing_response.response)

    except Exception, e:

        post_address_encoded = 'search_loc='+address.addr1.replace(' ', '+')+','+address.zipcode[:5]+'&token='+token+'&user='+str(user)+'&f=json'
        print(post_address_encoded)

        buffer = StringIO()
        c = pycurl.Curl()
        c.setopt(c.URL, 'https://cicero.azavea.com/v3.1/legislative_district')
        #c.setopt(c.REFERER, 'http://leveragecampaignfinance.org/')
        c.setopt(c.POST, True)

        c.setopt(c.WRITEDATA, buffer)

        c.setopt(c.POSTFIELDS, post_address_encoded)

        c.perform()
        c.close()

        result_json = buffer.getvalue()

        print result_json

        result = json.loads(result_json)

        cicero_raw = PoliticalDonationContributorAddressCiceroRaw()
        cicero_raw.addr1 = address.addr1
        cicero_raw.zipcode5 = address.zipcode[:5]
        cicero_raw.response = result_json

        session.add(cicero_raw)
        session.commit()

        return result


# Obtain a token:
if token == '' or user == '':
    postfields = {"username":os.environ.get('CICERO_USER'), "password":os.environ.get('CICERO_PASS')}
    response = get_cicero_response('http://cicero.azavea.com/v3.1/token/new.json', urlencode(postfields))
    print(response)

    try:
        response['success'] == True

        token = response['token']
        user = response['user']

    except Exception, e:

        print('Error getting token.', str(e))


#print('token:', token)
#print('user:', user)


processed_list = []


# Get addresses for committee 601 from state = 'PA'
addresses = session.query(PoliticalDonationContributorAddress)\
    .join(PoliticalDonationContributorAddress.contributors)\
    .join(PoliticalDonationContributor.donations)\
    .filter(PoliticalDonationContributorAddress.state == 'PA')\
    .filter(PoliticalDonation.committee_id == 601)

for address in addresses:

    if address.addr1+address.zipcode[:5] in processed_list:
        continue

    processed_list.append(address.addr1+address.zipcode[:5])

    cicero_objs = get_cicero_legislatrive_response(address, token, user)

    break