http_status_checker_gui.py

import sys
import asyncio
import json
import csv
from typing import Optional, Dict, Set, Tuple
from PySide6.QtWidgets import (
    QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
    QLabel, QLineEdit, QPushButton, QSpinBox, QProgressBar,
    QTextEdit, QFileDialog, QTabWidget, QTableWidget, QTableWidgetItem,
    QHeaderView, QMessageBox, QCheckBox, QComboBox
)
from PySide6.QtCore import Qt, QThread, Signal
from pathlib import Path
from datetime import datetime
from http_status_checker import AsyncHTTPStatusChecker
from urllib.parse import urlparse

class WorkerThread(QThread):
    progress_update = Signal(str)  # İlerleme mesajları için
    stage_update = Signal(str)     # Aşama değişiklikleri için
    stats_update = Signal(dict)    # İstatistik güncellemeleri için
    error_occurred = Signal(str)   # Hata mesajları için
    finished = Signal(dict)        # İşlem tamamlandığında sonuçlar için

    def __init__(self, sitemap_url, start_url, workers, batch_size, depth, output_dir):
        super().__init__()
        self.sitemap_url = sitemap_url
        self.start_url = start_url
        self.workers = workers
        self.batch_size = batch_size
        self.depth = depth
        self.output_dir = output_dir
        self.is_running = True

    def run(self):
        try:
            # Event loop'u bu thread'de oluştur
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            
            # Ana domain'i belirle
            base_domain = urlparse(self.sitemap_url or self.start_url).netloc.lower()
            if base_domain.startswith('www.'):
                base_domain = base_domain[4:]
                
            self.progress_update.emit(f"Ana domain: {base_domain}")
            
            # Checker'ı yapılandır
            checker = AsyncHTTPStatusChecker(
                base_domain=base_domain,
                max_workers=self.workers,
                batch_size=self.batch_size,
                output_dir=self.output_dir
            )
            
            # İlerleme bildirimi için callback'leri ayarla
            async def progress_callback(message):
                self.progress_update.emit(message)
                
            async def stage_callback(stage):
                self.stage_update.emit(stage)
                
            async def stats_callback(stats):
                self.stats_update.emit(stats)
            
            # Callback'leri checker'a bağla
            checker.set_callbacks(
                progress_callback=progress_callback,
                stage_callback=stage_callback,
                stats_callback=stats_callback
            )
            
            # Ana işlemi çalıştır
            async def main():
                try:
                    # AŞAMA 1: Sitemap Keşfi
                    if self.sitemap_url:
                        await checker.discover_sitemap_urls(self.sitemap_url)
                    
                    # AŞAMA 2: DOM Keşfi
                    if self.start_url:
                        await checker.crawl_for_dom_elements(self.start_url, self.depth)
                    
                    # AŞAMA 3: HTTP Durum Kodu Kontrolü
                    await checker.check_all_urls()
                    
                    # İstatistikleri kaydet
                    await checker.save_stats()
                    
                    # Sonuçları oku
                    with open(checker.stats_file, 'r', encoding='utf-8') as f:
                        stats = json.load(f)
                    
                    self.finished.emit(stats)
                    
                except Exception as e:
                    self.error_occurred.emit(str(e))
            
            # Event loop'u çalıştır
            loop.run_until_complete(main())
            
        except Exception as e:
            self.error_occurred.emit(str(e))
        finally:
            self.is_running = False
            if loop:
                loop.close()

    def stop(self):
        self.is_running = False
        self.terminate()

class MainWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("HTTP Status Kod Kontrol Aracı")
        self.setMinimumSize(800, 600)
        
        # Ana widget ve layout
        main_widget = QWidget()
        self.setCentralWidget(main_widget)
        layout = QVBoxLayout(main_widget)
        
        # Giriş alanları
        input_layout = QVBoxLayout()
        
        # Sitemap URL
        sitemap_layout = QHBoxLayout()
        sitemap_label = QLabel("Sitemap URL:")
        self.sitemap_input = QLineEdit()
        sitemap_layout.addWidget(sitemap_label)
        sitemap_layout.addWidget(self.sitemap_input)
        input_layout.addLayout(sitemap_layout)
        
        # Başlangıç URL
        start_url_layout = QHBoxLayout()
        start_url_label = QLabel("Başlangıç URL:")
        self.start_url_input = QLineEdit()
        start_url_layout.addWidget(start_url_label)
        start_url_layout.addWidget(self.start_url_input)
        input_layout.addLayout(start_url_layout)
        
        # Ayarlar
        settings_layout = QHBoxLayout()
        
        # Workers
        workers_label = QLabel("Workers:")
        self.workers_input = QSpinBox()
        self.workers_input.setRange(1, 100)
        self.workers_input.setValue(20)
        settings_layout.addWidget(workers_label)
        settings_layout.addWidget(self.workers_input)
        
        # Batch Size
        batch_label = QLabel("Batch Size:")
        self.batch_input = QSpinBox()
        self.batch_input.setRange(1, 200)
        self.batch_input.setValue(50)
        settings_layout.addWidget(batch_label)
        settings_layout.addWidget(self.batch_input)
        
        # Depth
        depth_label = QLabel("Derinlik:")
        self.depth_input = QSpinBox()
        self.depth_input.setRange(1, 10)
        self.depth_input.setValue(2)
        settings_layout.addWidget(depth_label)
        settings_layout.addWidget(self.depth_input)
        
        input_layout.addLayout(settings_layout)
        
        # Çıktı dizini
        output_layout = QHBoxLayout()
        output_label = QLabel("Çıktı Dizini:")
        self.output_input = QLineEdit()
        self.output_input.setText("output")
        output_button = QPushButton("Gözat")
        output_button.clicked.connect(self.select_output_dir)
        output_layout.addWidget(output_label)
        output_layout.addWidget(self.output_input)
        output_layout.addWidget(output_button)
        input_layout.addLayout(output_layout)
        
        layout.addLayout(input_layout)
        
        # Başlat/Durdur düğmesi
        self.start_button = QPushButton("Başlat")
        self.start_button.clicked.connect(self.toggle_crawl)
        layout.addWidget(self.start_button)
        
        # İlerleme çubuğu
        self.progress_bar = QProgressBar()
        self.progress_bar.setTextVisible(False)
        layout.addWidget(self.progress_bar)
        
        # Tab widget
        tabs = QTabWidget()
        
        # Log sekmesi
        self.log_text = QTextEdit()
        self.log_text.setReadOnly(True)
        tabs.addTab(self.log_text, "Log")
        
        # Sonuçlar sekmesi
        results_widget = QWidget()
        results_layout = QVBoxLayout(results_widget)
        
        # Filtre alanları
        filter_layout = QHBoxLayout()
        
        # Status Code filtresi
        status_group = QWidget()
        status_layout = QHBoxLayout(status_group)
        status_layout.setContentsMargins(0, 0, 0, 0)
        
        self.status_filters = {
            '2xx': QCheckBox('2xx (Başarılı)'),
            '3xx': QCheckBox('3xx (Yönlendirme)'),
            '4xx': QCheckBox('4xx (İstemci Hatası)'),
            '5xx': QCheckBox('5xx (Sunucu Hatası)'),
            '0xx': QCheckBox('0xx (Bağlantı Hatası)')
        }
        
        for checkbox in self.status_filters.values():
            checkbox.setChecked(True)
            checkbox.stateChanged.connect(self.apply_filters)
            status_layout.addWidget(checkbox)
            
        filter_layout.addWidget(status_group)
        
        # Arama filtresi
        search_group = QWidget()
        search_layout = QHBoxLayout(search_group)
        search_layout.setContentsMargins(0, 0, 0, 0)
        
        self.search_field = QComboBox()
        self.search_field.addItems(['URL', 'Title', 'H1', 'H2'])
        search_layout.addWidget(self.search_field)
        
        self.search_input = QLineEdit()
        self.search_input.setPlaceholderText('Ara...')
        self.search_input.textChanged.connect(self.apply_filters)
        search_layout.addWidget(self.search_input)
        
        filter_layout.addWidget(search_group)
        
        # Kaynak filtresi
        source_group = QWidget()
        source_layout = QHBoxLayout(source_group)
        source_layout.setContentsMargins(0, 0, 0, 0)
        
        self.source_filters = {
            'sitemap': QCheckBox('Sitemap'),
            'crawl': QCheckBox('Crawl')
        }
        
        for checkbox in self.source_filters.values():
            checkbox.setChecked(True)
            checkbox.stateChanged.connect(self.apply_filters)
            source_layout.addWidget(checkbox)
            
        filter_layout.addWidget(source_group)
        
        # İstatistik etiketi
        self.stats_label = QLabel()
        filter_layout.addWidget(self.stats_label)
        
        results_layout.addLayout(filter_layout)
        
        # Sonuçlar tablosu
        self.results_table = QTableWidget()
        self.results_table.setColumnCount(7)  # URL, Status, Title, H1, H2, Source, Date
        self.results_table.setHorizontalHeaderLabels(["URL", "Status", "Title", "H1", "H2", "Kaynak", "Tarih"])
        header = self.results_table.horizontalHeader()
        header.setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch)
        results_layout.addWidget(self.results_table)
        
        tabs.addTab(results_widget, "Sonuçlar")
        
        # DOM Elementleri sekmesi
        dom_widget = QWidget()
        dom_layout = QVBoxLayout(dom_widget)
        
        # DOM filtre alanları
        dom_filter_layout = QHBoxLayout()
        
        # Tag filtresi
        tag_group = QWidget()
        tag_layout = QHBoxLayout(tag_group)
        tag_layout.setContentsMargins(0, 0, 0, 0)
        
        self.tag_filters = {
            'a': QCheckBox('Links (a)'),
            'img': QCheckBox('Images (img)'),
            'link': QCheckBox('Stylesheets (link)'),
            'form': QCheckBox('Forms (form)')
        }
        
        for checkbox in self.tag_filters.values():
            checkbox.setChecked(True)
            checkbox.stateChanged.connect(self.apply_dom_filters)
            tag_layout.addWidget(checkbox)
            
        dom_filter_layout.addWidget(tag_group)
        
        # DOM arama filtresi
        dom_search_group = QWidget()
        dom_search_layout = QHBoxLayout(dom_search_group)
        dom_search_layout.setContentsMargins(0, 0, 0, 0)
        
        self.dom_search_field = QComboBox()
        self.dom_search_field.addItems(['URL', 'Text'])
        dom_search_layout.addWidget(self.dom_search_field)
        
        self.dom_search_input = QLineEdit()
        self.dom_search_input.setPlaceholderText('Ara...')
        self.dom_search_input.textChanged.connect(self.apply_dom_filters)
        dom_search_layout.addWidget(self.dom_search_input)
        
        dom_filter_layout.addWidget(dom_search_group)
        
        # DOM istatistik etiketi
        self.dom_stats_label = QLabel()
        dom_filter_layout.addWidget(self.dom_stats_label)
        
        dom_layout.addLayout(dom_filter_layout)
        
        # DOM tablosu
        self.dom_table = QTableWidget()
        self.dom_table.setColumnCount(5)  # Tag, Attribute, URL, Text, Source URL
        self.dom_table.setHorizontalHeaderLabels(["Tag", "Attribute", "URL", "Text", "Kaynak URL"])
        header = self.dom_table.horizontalHeader()
        header.setSectionResizeMode(2, QHeaderView.ResizeMode.Stretch)
        dom_layout.addWidget(self.dom_table)
        
        tabs.addTab(dom_widget, "DOM Elementleri")
        
        layout.addWidget(tabs)
        
        # Worker thread
        self.worker = None
        self.is_running = False

    def select_output_dir(self):
        dir_path = QFileDialog.getExistingDirectory(self, "Çıktı Dizinini Seç")
        if dir_path:
            self.output_input.setText(dir_path)

    def toggle_crawl(self):
        if not self.is_running:
            self.start_crawl()
        else:
            self.stop_crawl()

    def start_crawl(self):
        sitemap_url = self.sitemap_input.text().strip()
        start_url = self.start_url_input.text().strip()
        
        if not sitemap_url and not start_url:
            QMessageBox.warning(self, "Hata", "Sitemap URL veya başlangıç URL'i gerekli!")
            return
            
        self.is_running = True
        self.start_button.setText("Durdur")
        self.progress_bar.setMaximum(0)  # Belirsiz ilerleme
        self.log_text.clear()
        
        # Worker thread'i başlat
        self.worker = WorkerThread(
            sitemap_url=sitemap_url,
            start_url=start_url,
            workers=self.workers_input.value(),
            batch_size=self.batch_input.value(),
            depth=self.depth_input.value(),
            output_dir=self.output_input.text()
        )
        
        # Sinyalleri bağla
        self.worker.progress_update.connect(self.update_progress)
        self.worker.stage_update.connect(self.update_stage)
        self.worker.stats_update.connect(self.update_stats)
        self.worker.error_occurred.connect(self.handle_error)
        self.worker.finished.connect(self.handle_finished)
        
        # Thread'i başlat
        self.worker.start()

    def stop_crawl(self):
        if self.worker and self.worker.is_running:
            self.worker.stop()
            self.is_running = False
            self.start_button.setText("Başlat")
            self.progress_bar.setMaximum(100)
            self.progress_bar.setValue(0)
            self.log_text.append("İşlem durduruldu.")

    def update_progress(self, message):
        self.log_text.append(message)
        self.log_text.verticalScrollBar().setValue(
            self.log_text.verticalScrollBar().maximum()
        )

    def update_stage(self, stage):
        self.log_text.append(f"\n=== {stage} ===\n")

    def update_stats(self, stats):
        self.log_text.append("\nİstatistikler:")
        for key, value in stats.items():
            self.log_text.append(f"{key}: {value}")

    def handle_error(self, error_message):
        QMessageBox.critical(self, "Hata", error_message)
        self.stop_crawl()

    def handle_finished(self, stats):
        self.is_running = False
        self.start_button.setText("Başlat")
        self.progress_bar.setMaximum(100)
        self.progress_bar.setValue(100)
        
        # Sonuçları yükle
        self.load_results()
        self.load_dom_elements()
        
        QMessageBox.information(
            self,
            "Tamamlandı",
            f"İşlem tamamlandı!\n\n"
            f"Sitemap URL'leri: {stats.get('sitemap_urls', 0)}\n"
            f"Crawl URL'leri: {stats.get('crawled_urls', 0)}\n"
            f"DOM Elementleri: {sum(stats.get('dom_elements', {}).values())}"
        )
        
    def load_results(self):
        """results.csv dosyasından sonuçları yükle"""
        try:
            self.results_table.setRowCount(0)
            results = []
            
            with open('output/results.csv', 'r', encoding='utf-8') as f:
                reader = csv.DictReader(f)
                for row in reader:
                    results.append(row)
                    
            self.all_results = results  # Filtreleme için tüm sonuçları sakla
            self.apply_filters()
            
        except Exception as e:
            QMessageBox.warning(self, "Hata", f"Sonuçlar yüklenirken hata oluştu: {e}")
            
    def load_dom_elements(self):
        """dom_elements.csv dosyasından DOM elementlerini yükle"""
        try:
            self.dom_table.setRowCount(0)
            dom_elements = []
            
            with open('output/dom_elements.csv', 'r', encoding='utf-8') as f:
                reader = csv.DictReader(f)
                for row in reader:
                    dom_elements.append(row)
                    
            self.all_dom_elements = dom_elements  # Filtreleme için tüm elementleri sakla
            self.apply_dom_filters()
            
        except Exception as e:
            QMessageBox.warning(self, "Hata", f"DOM elementleri yüklenirken hata oluştu: {e}")
            
    def apply_dom_filters(self):
        """DOM elementleri için filtreleri uygula"""
        if not hasattr(self, 'all_dom_elements'):
            return
            
        filtered_elements = []
        total_count = len(self.all_dom_elements)
        
        for element in self.all_dom_elements:
            # Tag kontrolü
            tag = element.get('tag', '')
            if not self.tag_filters.get(tag, True).isChecked():
                continue
                
            # Metin araması
            search_text = self.dom_search_input.text().lower()
            if search_text:
                field = self.dom_search_field.currentText().lower()
                if field == 'url':
                    if search_text not in element.get('url', '').lower():
                        continue
                else:
                    if search_text not in element.get('text', '').lower():
                        continue
                        
            filtered_elements.append(element)
            
        # Tabloyu güncelle
        self.dom_table.setRowCount(len(filtered_elements))
        for i, element in enumerate(filtered_elements):
            self.dom_table.setItem(i, 0, QTableWidgetItem(element.get('tag', '')))
            self.dom_table.setItem(i, 1, QTableWidgetItem(element.get('attribute', '')))
            self.dom_table.setItem(i, 2, QTableWidgetItem(element.get('url', '')))
            self.dom_table.setItem(i, 3, QTableWidgetItem(element.get('text', '')))
            self.dom_table.setItem(i, 4, QTableWidgetItem(element.get('source_url', '')))
            
        # İstatistikleri güncelle
        stats = {}
        for tag in self.tag_filters.keys():
            stats[tag] = len([e for e in filtered_elements if e.get('tag') == tag])
            
        self.dom_stats_label.setText(
            f"Gösterilen: {len(filtered_elements)}/{total_count} | " +
            " | ".join(f"{tag}: {count}" for tag, count in stats.items())
        )
            
    def apply_filters(self):
        """Tüm filtreleri uygula"""
        if not hasattr(self, 'all_results'):
            return
            
        filtered_results = []
        total_count = len(self.all_results)
        
        for result in self.all_results:
            # Status code kontrolü
            status_code = int(result.get('status_code', 0))
            status_group = f"{status_code // 100}xx"
            
            if not self.status_filters.get(status_group, True).isChecked():
                continue
                
            # Kaynak kontrolü
            source = result.get('source', '')
            if not self.source_filters.get(source, True).isChecked():
                continue
                
            # Metin araması
            search_text = self.search_input.text().lower()
            if search_text:
                field = self.search_field.currentText().lower()
                if field == 'url':
                    if search_text not in result.get('url', '').lower():
                        continue
                else:
                    if search_text not in result.get(field.lower(), '').lower():
                        continue
                        
            filtered_results.append(result)
            
        # Tabloyu güncelle
        self.results_table.setRowCount(len(filtered_results))
        for i, result in enumerate(filtered_results):
            self.results_table.setItem(i, 0, QTableWidgetItem(result.get('url', '')))
            
            # Status code'a göre renk
            status_item = QTableWidgetItem(str(result.get('status_code', '')))
            status_code = int(result.get('status_code', 0))
            if status_code >= 200 and status_code < 300:
                status_item.setBackground(Qt.GlobalColor.green)
            elif status_code >= 300 and status_code < 400:
                status_item.setBackground(Qt.GlobalColor.yellow)
            elif status_code >= 400:
                status_item.setBackground(Qt.GlobalColor.red)
            self.results_table.setItem(i, 1, status_item)
            
            self.results_table.setItem(i, 2, QTableWidgetItem(result.get('title', '')))
            self.results_table.setItem(i, 3, QTableWidgetItem(result.get('h1', '')))
            self.results_table.setItem(i, 4, QTableWidgetItem(result.get('h2', '')))
            self.results_table.setItem(i, 5, QTableWidgetItem(result.get('source', '')))
            self.results_table.setItem(i, 6, QTableWidgetItem(result.get('check_date', '')))
            
        # İstatistikleri güncelle
        stats = {
            '2xx': len([r for r in filtered_results if 200 <= int(r.get('status_code', 0)) < 300]),
            '3xx': len([r for r in filtered_results if 300 <= int(r.get('status_code', 0)) < 400]),
            '4xx': len([r for r in filtered_results if 400 <= int(r.get('status_code', 0)) < 500]),
            '5xx': len([r for r in filtered_results if 500 <= int(r.get('status_code', 0)) < 600]),
            '0xx': len([r for r in filtered_results if int(r.get('status_code', 0)) == 0])
        }
        
        self.stats_label.setText(
            f"Gösterilen: {len(filtered_results)}/{total_count} | "
            f"2xx: {stats['2xx']} | "
            f"3xx: {stats['3xx']} | "
            f"4xx: {stats['4xx']} | "
            f"5xx: {stats['5xx']} | "
            f"Hata: {stats['0xx']}"
        )

def main():
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec())

if __name__ == '__main__':
    main()