|
| 1 | +import sys |
| 2 | +import os |
| 3 | +import sysconfig |
| 4 | +import re |
| 5 | +import requests |
| 6 | +from packaging.version import parse as parse_version, InvalidVersion |
| 7 | +from PySide6.QtWidgets import ( |
| 8 | + QApplication, QWidget, QVBoxLayout, QPushButton, QFileDialog, |
| 9 | + QMessageBox, QProgressBar, QCheckBox, QTextEdit |
| 10 | +) |
| 11 | +from PySide6.QtCore import Qt, QThread, Signal |
| 12 | + |
| 13 | + |
| 14 | +# -------------------- HELPERS -------------------- |
| 15 | + |
| 16 | +def get_standard_libs(): |
| 17 | + std_lib = sysconfig.get_paths()["stdlib"] |
| 18 | + std_libs = set() |
| 19 | + for root, _, files in os.walk(std_lib): |
| 20 | + for file in files: |
| 21 | + if file.endswith(".py"): |
| 22 | + rel_path = os.path.relpath(os.path.join(root, file), std_lib) |
| 23 | + module = rel_path.replace(os.sep, ".").rsplit(".py", 1)[0] |
| 24 | + std_libs.add(module.split(".")[0]) |
| 25 | + return std_libs |
| 26 | + |
| 27 | + |
| 28 | +def clean_and_merge_requirements(reqs, log_fn): |
| 29 | + clean_reqs = {} |
| 30 | + pattern = re.compile(r"^([A-Za-z0-9_.\-]+)\s*([=<>!~]*.*)?$") |
| 31 | + |
| 32 | + for r in reqs: |
| 33 | + r = r.strip() |
| 34 | + if not r or r.startswith("#") or "@ file://" in r: |
| 35 | + continue |
| 36 | + |
| 37 | + match = pattern.match(r) |
| 38 | + if not match: |
| 39 | + log_fn(f"Skipping invalid line: {r}") |
| 40 | + continue |
| 41 | + |
| 42 | + pkg, spec = match.groups() |
| 43 | + pkg = pkg.lower() |
| 44 | + spec = spec.strip() if spec else "" |
| 45 | + |
| 46 | + if pkg in clean_reqs: |
| 47 | + old_spec = clean_reqs[pkg] |
| 48 | + try: |
| 49 | + if "==" in spec: |
| 50 | + new_ver = spec.split("==")[-1] |
| 51 | + old_ver = old_spec.split("==")[-1] if "==" in old_spec else "" |
| 52 | + if not old_ver or parse_version(new_ver) > parse_version(old_ver): |
| 53 | + clean_reqs[pkg] = spec |
| 54 | + else: |
| 55 | + clean_reqs[pkg] = old_spec or spec |
| 56 | + except InvalidVersion: |
| 57 | + log_fn(f"Invalid version format for {pkg}: {spec}") |
| 58 | + clean_reqs[pkg] = spec or old_spec |
| 59 | + else: |
| 60 | + clean_reqs[pkg] = spec |
| 61 | + |
| 62 | + return [f"{pkg}{spec}" if spec else pkg for pkg, spec in sorted(clean_reqs.items())] |
| 63 | + |
| 64 | + |
| 65 | +def validate_on_pypi(requirements, log_fn): |
| 66 | + valid_reqs = [] |
| 67 | + for line in requirements: |
| 68 | + pkg = re.split(r"[=<>!~]", line)[0].strip() |
| 69 | + url = f"https://pypi.org/pypi/{pkg}/json" |
| 70 | + try: |
| 71 | + resp = requests.get(url, timeout=2) |
| 72 | + if resp.status_code == 200: |
| 73 | + valid_reqs.append(line) |
| 74 | + else: |
| 75 | + log_fn(f"Package not on PyPI: {pkg}") |
| 76 | + except Exception: |
| 77 | + log_fn(f"Could not validate package: {pkg}") |
| 78 | + return valid_reqs |
| 79 | + |
| 80 | + |
| 81 | +def safe_read_file(file_path, log_fn): |
| 82 | + for enc in ["utf-8-sig", "utf-8", "latin-1"]: |
| 83 | + try: |
| 84 | + with open(file_path, "r", encoding=enc, errors="ignore") as f: |
| 85 | + return f.readlines() |
| 86 | + except Exception: |
| 87 | + continue |
| 88 | + log_fn(f"Could not read file: {file_path}") |
| 89 | + return [] |
| 90 | + |
| 91 | + |
| 92 | +# -------------------- WORKER THREAD -------------------- |
| 93 | + |
| 94 | +class Worker(QThread): |
| 95 | + progress = Signal(int) |
| 96 | + finished = Signal(list) |
| 97 | + log_msg = Signal(str) |
| 98 | + |
| 99 | + def __init__(self, source_dir, exclude_std): |
| 100 | + super().__init__() |
| 101 | + self.source_dir = source_dir |
| 102 | + self.exclude_std = exclude_std |
| 103 | + self.std_libs = get_standard_libs() if exclude_std else set() |
| 104 | + |
| 105 | + def log(self, message): |
| 106 | + self.log_msg.emit(message) |
| 107 | + |
| 108 | + def run(self): |
| 109 | + requirements = set() |
| 110 | + all_files = [] |
| 111 | + for root, _, files in os.walk(self.source_dir): |
| 112 | + for file in files: |
| 113 | + if file.endswith(".py") or file == "requirements.txt": |
| 114 | + all_files.append(os.path.join(root, file)) |
| 115 | + |
| 116 | + total_files = len(all_files) |
| 117 | + for idx, file_path in enumerate(all_files): |
| 118 | + if file_path.endswith(".py"): |
| 119 | + self.process_python_file(file_path, requirements) |
| 120 | + elif file_path.endswith("requirements.txt"): |
| 121 | + self.process_requirements_file(file_path, requirements) |
| 122 | + self.progress.emit(int((idx + 1) / total_files * 100)) |
| 123 | + |
| 124 | + if self.exclude_std: |
| 125 | + requirements = {pkg for pkg in requirements if pkg not in self.std_libs} |
| 126 | + |
| 127 | + cleaned = clean_and_merge_requirements(requirements, self.log) |
| 128 | + validated = validate_on_pypi(cleaned, self.log) |
| 129 | + |
| 130 | + self.finished.emit(validated) |
| 131 | + |
| 132 | + def process_python_file(self, file_path, requirements): |
| 133 | + lines = safe_read_file(file_path, self.log) |
| 134 | + for i, line in enumerate(lines): |
| 135 | + if i >= 50: |
| 136 | + break |
| 137 | + line = line.strip() |
| 138 | + if line.startswith("import "): |
| 139 | + pkg = line.split()[1].split(".")[0] |
| 140 | + requirements.add(pkg) |
| 141 | + elif line.startswith("from "): |
| 142 | + pkg = line.split()[1].split(".")[0] |
| 143 | + requirements.add(pkg) |
| 144 | + |
| 145 | + def process_requirements_file(self, file_path, requirements): |
| 146 | + lines = safe_read_file(file_path, self.log) |
| 147 | + for line in lines: |
| 148 | + line = line.strip() |
| 149 | + if line and not line.startswith("#"): |
| 150 | + requirements.add(line) |
| 151 | + |
| 152 | + |
| 153 | +# -------------------- GUI -------------------- |
| 154 | + |
| 155 | +class RequirementsCollector(QWidget): |
| 156 | + def __init__(self): |
| 157 | + super().__init__() |
| 158 | + self.setWindowTitle("Master Requirements Generator") |
| 159 | + self.setGeometry(200, 200, 500, 400) |
| 160 | + |
| 161 | + layout = QVBoxLayout() |
| 162 | + |
| 163 | + self.select_src_btn = QPushButton("Select Source Directory") |
| 164 | + self.select_src_btn.clicked.connect(self.select_source_dir) |
| 165 | + layout.addWidget(self.select_src_btn) |
| 166 | + |
| 167 | + self.select_dest_btn = QPushButton("Select Destination for Master Requirements") |
| 168 | + self.select_dest_btn.clicked.connect(self.select_dest_file) |
| 169 | + layout.addWidget(self.select_dest_btn) |
| 170 | + |
| 171 | + self.exclude_std_cb = QCheckBox("Exclude standard libraries") |
| 172 | + layout.addWidget(self.exclude_std_cb) |
| 173 | + |
| 174 | + |
| 175 | + self.log_box = QTextEdit() |
| 176 | + self.log_box.setReadOnly(True) |
| 177 | + layout.addWidget(self.log_box) |
| 178 | + |
| 179 | + self.generate_btn = QPushButton("Generate Master Requirements") |
| 180 | + self.generate_btn.clicked.connect(self.generate_requirements) |
| 181 | + layout.addWidget(self.generate_btn) |
| 182 | + |
| 183 | + self.setLayout(layout) |
| 184 | + |
| 185 | + self.source_dir = "" |
| 186 | + self.dest_file = "" |
| 187 | + |
| 188 | + def select_source_dir(self): |
| 189 | + dir_path = QFileDialog.getExistingDirectory(self, "Select Source Directory") |
| 190 | + if dir_path: |
| 191 | + self.source_dir = dir_path |
| 192 | + |
| 193 | + def select_dest_file(self): |
| 194 | + file_path, _ = QFileDialog.getSaveFileName(self, "Save Master Requirements", "requirements.txt", "Text Files (*.txt)") |
| 195 | + if file_path: |
| 196 | + self.dest_file = file_path |
| 197 | + |
| 198 | + def generate_requirements(self): |
| 199 | + if not self.source_dir or not self.dest_file: |
| 200 | + QMessageBox.warning(self, "Error", "Please select both source directory and destination file.") |
| 201 | + return |
| 202 | + |
| 203 | + self.worker = Worker(self.source_dir, self.exclude_std_cb.isChecked()) |
| 204 | + # Progress bar removed |
| 205 | + self.worker.log_msg.connect(self.log) |
| 206 | + self.worker.finished.connect(self.write_requirements) |
| 207 | + self.worker.start() |
| 208 | + |
| 209 | + def log(self, message): |
| 210 | + self.log_box.append(message) |
| 211 | + |
| 212 | + def write_requirements(self, requirements): |
| 213 | + try: |
| 214 | + with open(self.dest_file, "w") as f: |
| 215 | + for req in requirements: |
| 216 | + f.write(req + "\n") |
| 217 | + QMessageBox.information(self, "Success", f"Master requirements.txt created at:\n{self.dest_file}") |
| 218 | + except Exception as e: |
| 219 | + QMessageBox.critical(self, "Error", f"Could not write file: {e}") |
| 220 | + |
| 221 | + |
| 222 | +# -------------------- MAIN -------------------- |
| 223 | + |
| 224 | +if __name__ == "__main__": |
| 225 | + app = QApplication(sys.argv) |
| 226 | + window = RequirementsCollector() |
| 227 | + window.show() |
| 228 | + sys.exit(app.exec()) |
0 commit comments