parsing_asset

# -*- coding: utf-8 -*-
"""
Created on Thu Aug 13 15:19:08 2020

@author: user

감사보고서일 추출 코드 - New report format

"""

from bs4 import BeautifulSoup
import os
import glob
import pandas as pd
import re

def col_span_count(soup):
    try:
        result = int(soup["colspan"])
    except KeyError:
        result= 1
    return result

def row_span_count(soup):
    try:
        result = int(soup["rowspan"])
    except KeyError:
        result= 1
    return result

def FindTargetTable(soup):
    tables = soup.find_all("table")
    # for i in tables:
    #     tds = i.find_all("td")
    #     if len(tds) > 20:  # 시간 파트가 약 80정도 된다. 비교 표시 없는 경우 td가 50도 안된다. 첫번째 나오는 20 초과 table 선택
    #         break
    return tables

def MatrixGenerator(table):
    table_row = table.find_all("tr")

    columnCount = 0
    for i in table_row:
        columnNumber = 0
        for j in i.find_all(["th", "td"]):
            try:
                columnNumber += int(j["colspan"])
            except KeyError:
                columnNumber += 1
            if columnNumber > columnCount:
                columnCount = columnNumber        
    rowCount = len(table_row)
    matrix = [['#' for x in range(columnCount)] for y in range(rowCount)] 
    for i in range(len(table_row)):
        locator = [i for i, x in enumerate(matrix[i]) if x=='#']  # https://stackoverflow.com/questions/9542738/python-find-in-list
        column, colSpan = 0, 0
        for j in table_row[i].find_all(["th", "td"]):
            rowSpanCount = row_span_count(j)
            colSpanCount = col_span_count(j)

            for k in range(rowSpanCount):
                for l in range(colSpanCount):
                    row = i + k
                    column = locator[l+colSpan]
                    matrix[row][column] = ''.join(j.text.replace("\xa0", "").strip().split())
            colSpan += col_span_count(j)
    return matrix

def ParsingTime(matrix, isComparative, tableLength, file, report):
    if isComparative:
        container = []
        tableLength = int(tableLength / 2)
        for i in range(tableLength):
            container.append(file + "_" + matrix[report][1] + "_" + matrix[1][2+2*i] + "_" + matrix[report][2+2*i].replace('-', '0').replace(',','') + "\n")
    else:
        container = []
        tableLength = int(tableLength)
        for i in range(tableLength):
            container(file + "_" + matrix[report][1] + "_" + matrix[1][2+i] + "_" + matrix[report][2+i].replace('-', '0').replace(',','') + "\n")
    return container

def Indexing(matrix):
    """
    당기, 전기 부분 삭제한 경우들이 있어서 찾아야함
    """
    container = []
    for i in matrix:
        try:
            i[0:2].index("투입 인원수")
            container.append(matrix.index(i))
        except ValueError:
            pass
        try:
            i[0:2].index("분ㆍ반기검토")
            container.append(matrix.index(i))
        except ValueError:
            pass
        try:
            i[0:2].index("감사")
            container.append(matrix.index(i))
        except ValueError:
            pass
        try:
            i[0:2].index("합계")
            container.append(matrix.index(i))
        except ValueError:
            pass
    return container

# 1. 작업 폴더로 변경
os.chdir("C:\\Users\\yoont\\Desktop\\wd\\")  # 작업 폴더로 변경

# 2. 타겟 폴더에 있는 필요 문서 경로 리스트업
pathList = []
for path in [".\A001_2019\\", 
             ".\F001_2019\\"]:
    path = path + "*감사보고서_(첨부)재무제표*.*"  # 필요한 Keyword 입력
    pathInProcess = glob.glob(path)
    pathList= pathList + pathInProcess

# 3. 입수 과정에서 중복입수되어 표시된 duplicated 표시 파일 제거
pathList = [x for x in pathList if "duplicated" not in x]

# 4. 연결감사보고서 첨부 문서 제거 (외부감사실시내용/사업보고서 optional)
pathList = [x for x in pathList if "연결감사보고서" not in x]

# 5. '[정정]'이 포함된 문서 제거
pathList = [x for x in pathList if "[정정]" not in x]

# 6. 분리
pathList = [x for x in pathList if "(2018.12)" in x]

PathListDf = pd.DataFrame(pathList)
df = pd.DataFrame([x.split("_") for x in pathList])
df["path"] = PathListDf[0]
df["key"] = df[2] + df[6].str.slice(stop=10) + df[5] + df[8] + df[10]
df["duplc"] = df.duplicated(subset=["key"], keep=False)
isTrue = df[df["duplc"] == True]
dataOut = df.drop_duplicates(subset=["key"])

pathListOut = dataOut["path"].tolist()

# Long data 입수 
directory = r"C:\\Users\\yoont\\Desktop\\"
workInProcess = directory + "total_asset.txt"

txtName = os.path.join(directory, workInProcess)
result = open(txtName, 'w', encoding="utf-8")

progress = 0

for file in pathListOut:

    html = open(file, "r", encoding="utf-8")
    soup = BeautifulSoup(html, "lxml")
    html.close()
    
    for td in soup.find_all("td"):
        if ''.join(td.text.split()).find("단위") > 0:
            unit = ''.join(td.text.split())
            if unit.find(":원") > 0:
                unit = '1'
            elif unit.find(":천원") > 0:
                unit = '1000'
            elif unit.find(":백만원") > 0:
                unit = '1000000'
            else:
                unit = "NA"           
            break
    
    for table in soup.find_all("table"):
        if ''.join(table.text.split()).find("부채") > 0:
            break
        table = BeautifulSoup("<table></table>", features="lxml").table  # 의견 거절 등 BS가 안붙어있을 때 처리
    
    matrix = MatrixGenerator(table)
    
    # 부채와자본총계(=자산총계) Parsing
    
    if len(matrix) > 0:  # BS가 있을 때
        i = matrix[len(matrix)-1]
        if len(matrix[0]) % 2 == 1:  # 주석 Column이 없을 때
            if i[1]:
                bsLine = [i[0], unit, i[1].replace("=", "")]
            elif i[2]:
                bsLine = [i[0], unit, i[2].replace("=", "")]
        
        else:  # 주석 Column이 있을 때
            if i[2]:
                bsLine = [i[0], unit, i[2].replace("=", "")]
            elif i[3]:
                bsLine = [i[0], unit, i[3].replace("=", "")]
        
        resultString = "_".join(bsLine)
    elif len(matrix) == 0:  # BS가 없을 때
        resultString = ""
        
    """
    # 전체 BS 당기 숫자 Parsing
    
    bsLineList = []
    
    if len(matrix[0]) % 2 == 1:  # 주석 Column이 없을 때
        for i in matrix:
            if i[1]:
                bsLine = [i[0], "E", unit, i[1].replace("=", "")]
            elif i[2]:
                bsLine = [i[0], "A", unit, i[2].replace("=", "")]
            bsLineList.append(bsLine)
    
    else:  # 주석 Column이 있을 때
        for i in matrix:
            if i[2]:
                bsLine = [i[0], "E", unit, i[2].replace("=", "")]
            elif i[3]:
                bsLine = [i[0], "A", unit, i[3].replace("=", "")]
            bsLineList.append(bsLine)
    
    print(bsLineList)
    """

    returnText = file + '_' + resultString + "\n"
    result.write(returnText)
    
    print('.', end='')

result.close()