feat: add support for local parquet files in Duck application

Author: lfoppiano

Makefile

@@ -44,6 +44,12 @@ duck_ccf_local_files: build
 	@echo "warning! only works on Common Crawl Foundation's development machine"
 	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.Duck -Dexec.args="ccf_local_files"
 
+duck_local_files: build
+	ifndef LOCAL_DIR
+		$(error LOCAL_DIR is required. Usage: make duck_local_files LOCAL_DIR=/path/to/data)
+	endif
+	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.Duck -Dexec.args="local_files $(LOCAL_DIR)"
+
 duck_cloudfront: build
 	@echo "warning! this might take 1-10 minutes"
 	mvn -q exec:java -Dexec.mainClass=org.commoncrawl.whirlwind.Duck -Dexec.args="cloudfront"

README.md

@@ -796,6 +796,14 @@ If you want to run many of these queries, and you have a lot of disk space, you'
 aws s3 sync s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-22/subset=warc/ .'
 ```
 
+or (if you don't have access through the AWS CLI): 
+
+```shell
+TBA - add the download instructions 
+```
+
+then you can run `make duck_local_files LOCAL_DIR=/path/to/the/downloaded/data` to run the same query as above, but this time using your local copy of the index files.
+
 > [!IMPORTANT]
 > If you happen to be using the Common Crawl Foundation development server, we've already downloaded these files, and you can run ```make duck_ccf_local_files```
 

src/main/java/org/commoncrawl/whirlwind/Duck.java

@@ -24,6 +24,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.*;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -38,7 +39,7 @@ public class Duck {
 	private static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
 
 	public enum Algorithm {
-		CCF_LOCAL_FILES("ccf_local_files"), CLOUDFRONT("cloudfront");
+		CCF_LOCAL_FILES("ccf_local_files"), CLOUDFRONT("cloudfront"), LOCAL_FILES("local_files");
 
 		private final String name;
 
@@ -113,8 +114,13 @@ public static void printRowAsKvList(ResultSet rs, PrintStream out) throws SQLExc
 	/**
 	 * Gets the list of parquet files to query based on the algorithm.
 	 */
-	public static List<String> getFiles(Algorithm algo, String crawl) throws IOException {
+	public static List<String> getFiles(Algorithm algo, String crawl, String localPrefix) throws IOException {
 		switch (algo) {
+		case LOCAL_FILES: {
+			Path indexPath = Path.of(localPrefix);
+			return getLocalParquetFiles(indexPath);
+		}
+
 		case CCF_LOCAL_FILES: {
 			Path indexPath = Path.of("/home/cc-pds/commoncrawl/cc-index/table/cc-main/warc", "crawl=" + crawl,
 					"subset=warc");
@@ -142,6 +148,23 @@ public static List<String> getFiles(Algorithm algo, String crawl) throws IOExcep
 		}
 	}
 
+	private static List<String> getLocalParquetFiles(Path indexPath) throws IOException {
+		if (!Files.isDirectory(indexPath)) {
+			System.err.println("Directory not found: " + indexPath);
+			System.exit(1);
+		}
+
+		List<String> files = Files.list(indexPath).filter(p -> p.toString().endsWith(".parquet")).map(Path::toString)
+				.collect(Collectors.toList());
+
+		if (files.isEmpty()) {
+			System.err.println("No parquet files found in: " + indexPath);
+			System.exit(1);
+		}
+
+		return files;
+	}
+
 	private static List<String> getLocalParquetFiles(Path indexPath, String prefix, String crawl) throws IOException {
 		if (!Files.isDirectory(indexPath)) {
 			printIndexDownloadAdvice(prefix, crawl);
@@ -189,6 +212,7 @@ private static ResultSet executeWithRetry(Statement stmt, String sql) throws SQL
 	public static void main(String[] args) {
 		String crawl = "CC-MAIN-2024-22";
 		Algorithm algo = Algorithm.CLOUDFRONT;
+		String localPrefix = "/home/cc-pds/commoncrawl/cc-index/table/cc-main/warc";
 
 		if (args.length > 0) {
 			if ("help".equalsIgnoreCase(args[0]) || "--help".equals(args[0]) || "-h".equals(args[0])) {
@@ -200,20 +224,30 @@ public static void main(String[] args) {
 			System.out.println("Using algorithm: " + algo.getName());
 		}
 
+		if (algo == Algorithm.LOCAL_FILES) {
+			if (args.length < 2) {
+				System.err.println("Error: local_files algorithm requires a directory argument.");
+				printUsage();
+				System.exit(1);
+			}
+			localPrefix = args[1];
+		}
+
 		try {
-			run(algo, crawl);
+			run(algo, crawl, localPrefix);
 		} catch (Exception e) {
 			System.err.println("Error: " + e.getMessage());
 			printUsage();
 			System.exit(1);
 		}
 	}
 
-	public static void run(Algorithm algo, String crawl) throws IOException, SQLException, InterruptedException {
+	public static void run(Algorithm algo, String crawl, String localPrefix)
+			throws IOException, SQLException, InterruptedException {
 		// Ensure stdout uses UTF-8
 		PrintStream out = new PrintStream(System.out, true, StandardCharsets.UTF_8);
 
-		List<String> files = getFiles(algo, crawl);
+		List<String> files = getFiles(algo, crawl, localPrefix);
 		String filesList = files.stream().map(f -> "'" + f + "'").collect(Collectors.joining(", "));
 
 		// Use in-memory DuckDB
@@ -304,14 +338,19 @@ private static void printResultSet(ResultSet rs, PrintStream out) throws SQLExce
 	}
 
 	private static void printUsage() {
-		System.err.println("Usage: Duck [algorithm]");
+		System.err.println("Usage: Duck [algorithm] [local-directory]");
 		System.err.println();
 		System.err.println("Query Common Crawl index using DuckDB.");
 		System.err.println();
 		System.err.println("Algorithms:");
-		System.err.println("  ccf_local_files  Use local parquet files from /home/cc-pds/commoncrawl/...");
+		System.err.println("  local_files      Use local parquet files (from specified local directory)");
+		System.err.println(
+				"  ccf_local_files  Use local parquet files (default: /home/cc-pds/commoncrawl/cc-index/table/cc-main/warc)");
 		System.err.println("  cloudfront       Use CloudFront URLs (requires <crawl>.warc.paths.gz file)");
 		System.err.println();
+		System.err.println("Arguments:");
+		System.err.println("  local-directory  Local directory prefix for 'local_files' algorithm");
+		System.err.println();
 		System.err.println("Options:");
 		System.err.println("  help, --help, -h  Show this help message");
 	}