threads/threads2.py at master · nytlabs/threads · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
import json
import collections
import datetime
import pylab

def articles2events(articles):
    events = [int(a['published']['$date']/1000) for a in articles]
    events.sort()
    return events

def avgInterval(events):
    if len(events) == 1:
        return None
    if len(events) == 0:
        raise ValueError("no events specified")
    intervals = [events[i+1] - events[i] for i in range(len(events)-1)]
    return sum(intervals)/len(intervals)

def getWindow(events, start, stop):
    out = [e for e in events if e > start and e < stop]
    return out

def slideWindow(events, windowSize, slideInterval):
    t0 = min(events)
    T = max(events)
    t = t0
    X = []
    while t + windowSize < T:
        w = getWindow(events, t, t+windowSize)
        if len(w) == 0:
            x = 0
        else:
            x = avgInterval(w)
        if x:
            l = 1./(x/86400.0)
        else:
            l = 0
        X.append((datetime.datetime.fromtimestamp(t),l))
        t += windowSize
    return X

articles = [json.loads(line) for line in open('articles.json')]

lookup = collections.defaultdict(list)
for a in articles:
    for k in a['keywords']:
        lookup[k['content'].lower()].append(a)

article2keywords = {}
for a in articles:
    article2keywords[a['_id']] = [k['content'].lower() for k in a['keywords']]

id2article = {}
for a in articles:
    id2article[a["_id"]] = a

keywords = lookup.keys()
keywords.sort()

threads = []
threadKeywords = []

for keyword in keywords:
    events = articles2events(lookup[keyword])
    ts = slideWindow(events, 604800, 17280)

    numpoints = len([y for x,y in ts if y > 0])
    if numpoints < 9: # need 20 non-zero points
        #if keyword.startswith("uk"):
        #    print "skipping", keyword, "as less than 20 total (%s)"%numpoints
        continue

    x,y = zip(*ts)

    if max(y) < 2: # articles per day
        print "skipping", keyword, "as less than 2 per day"
        continue

    #if sum(y) < 31: # total articles in period
    #    print "skipping", keyword, "as less than 31 articles total (%s)"%sum(y)
    #    continue
    if len(lookup[keyword]) < 60:
        print "skipping", keyword, "as less than 60 articles total (%s)"%len(lookup[keyword])
        continue

    # normalise

    yn = pylab.array(y)
    yn -= min(yn)
    yn /= pylab.mean(yn)

    if (max(yn) - pylab.mean(yn)) < 2.2: # no outliers
        #print "skipping", keyword, "as no outliers"
        continue

    threads.append(lookup[keyword])
    threadKeywords.append(keyword)

threadIDs = [set([article["_id"] for article in thread]) for thread in threads]

# merge overlapping threads

toMerge = collections.defaultdict(list)
for i,ti in enumerate(threadIDs):
    for j,tj in enumerate(threadIDs):
        if i == j:
            continue
        if len(ti.intersection(tj)) > len(threadIDs[i])/3.0:
            toMerge[i].append(j)

mergedThreads = []
mergedKeywordIdxs = []
mergedKeywords = []
for i in toMerge:
    if i in mergedKeywordIdxs:
        continue
    for j in toMerge[i]:
        mergedThreads.append(threadIDs[i].union(threadIDs[j]))
        mergedKeywordIdxs.append(j)
    mergedKeywords.append(threadKeywords[i])
    mergedKeywordIdxs.append(i)


# remove merged threads
mergedKeywordIdxs.sort()
mergedKeywordIdxs.reverse()
for idx in mergedKeywordIdxs:
    threadKeywords.pop(idx)
    threadIDs.pop(idx)

# join merged and non-merged
threadIDs += mergedThreads
threadKeywords += mergedKeywords

print threadKeywords

# form final JSON

outThreads = []
outKeywords = []
for thread in threadIDs:

    keywords = list()
    for j in thread:
        thisKeywords = article2keywords[j]
        for k in thisKeywords:
            keywords.append(k)

    keywordsCounter = collections.Counter(keywords)

    mostCommon = dict(keywordsCounter.most_common(3))
    if set(mostCommon.keys()) in outKeywords:
        print "skipping", mostCommon
        continue
    outKeywords.append(set(mostCommon.keys()))
    t = {
        "articles": [
            id2article[j]
            for j in thread
            if set(mostCommon.keys()).issubset(set(article2keywords[j]))
        ],
        "keywords": mostCommon
    }


    if len(set([a['headline'] for a in t['articles']])) < 10:
        continue

    if len(t['articles']) > 10:
        outThreads.append(t)

json.dump(outThreads, open("display/threads.json",'w'))