scrape-cirt_net_passwords

#!/usr/bin/env python2.7

import sys
import requests
from lxml.html import fromstring

def clean(content):
    if content is None:
        return ''
    content = content.strip()
    if content.lower() == 'n/a' or content.lower() == 'n.a' or content.lower() == '(none)' or content.lower() == '(blank)':
        content = ''
    if content.lower() == '(any 3 chars)':
        content = 'abc'
    return content

def clean_protocol(content):
    content = clean(content)
    content = content.replace(',', '/')
    content = content.replace(', ', '/')
    return content

def ignore(protocol, username, password):
    if password is not None:
        if 'ast 6 characters' in password:
            return True
        if 'characters of the' in password:
            return True
        if '0 and the number of OCE printer' in password:
            return True
        if 'use ALT+G' in password:
            return True
        if 'last eight digits' in password:
            return True
    return False

response = requests.get('https://cirt.net/passwords')
page = fromstring(response.text)
links = set()
for a in page.xpath("//a"):
    href = a.get("href")
    if href.startswith('?vendor='):
        links.add('https://cirt.net/passwords%s' % href)

for link in links:
    response = requests.get(link)
    page = fromstring(response.text)
    for table in page.xpath("//table"):
        protocol = None
        username = None
        password = None
        for row in table.findall("tr"):
            try:
                header = row[0].text_content()
                data = row[1].text_content()
            except IndexError:
                continue
            if header.strip().lower() == 'user id':
                username = data
            elif header.strip().lower() == 'password':
                password = data
            elif header.strip().lower() == 'method':
                protocol = data.lower() if data is not None else ''

        if ignore(protocol, username, password):
            continue

        if protocol is not None or username is not None or password is not None:
            try:
                sys.stdout.write('%s,%s,%s\n' % (clean_protocol(protocol), clean(username), clean(password)))
                sys.stdout.flush()
            except UnicodeEncodeError:
                continue