diff --git a/.gitignore b/.gitignore index 40005ba..d98e537 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,13 @@ /shard.lock /http_proxy +/.mitm-certs/ + +# Local MITM certificates/keys (generated for manual testing) +/mitm-ca.crt +/mitm-ca.key +/mitm-ca.srl +/mitm.crt +/mitm.key +/mitm.csr +/mitm.ext diff --git a/MITM_CHECKLIST.md b/MITM_CHECKLIST.md new file mode 100644 index 0000000..1089209 --- /dev/null +++ b/MITM_CHECKLIST.md @@ -0,0 +1,65 @@ +# MITM Production Readiness Checklist + +Status key: + +- [x] Done +- [ ] Not done +- [~] Partial + +## Security + +- [ ] Store CA private key outside repository with strict permissions (`600`) and dedicated service account. +- [ ] Add CA key rotation procedure and documented migration plan. +- [ ] Add encrypted CA key / passphrase support. +- [~] Keep sensitive logging disabled by default (`debug: false`), but add redaction for auth/cookies/tokens when debug is enabled. +- [ ] Add host allowlist/denylist policy for MITM interception. + +## Certificate Generation + +- [x] Dynamic per-host certificate generation is implemented. +- [x] In-process generator is implemented in `src/http/proxy/server/certificate_generator.cr`. +- [x] Certificate cache directory support exists (`certificate_cache_dir`). +- [x] Basic SAN generation exists for DNS/IP hostnames. +- [ ] Serial number strategy should be strengthened (currently timestamp-based). +- [ ] Add cert cache lifecycle controls (TTL/max size/pruning). + +## Protocol Behavior + +- [x] CONNECT MITM flow for HTTP/1.1 is implemented. +- [x] ALPN is constrained to HTTP/1.1 in server contexts. +- [~] Non-HTTP streams are detected and aborted in MITM path; consider explicit passthrough/tunnel fallback policy. +- [ ] Define and document policy for HTTP/2/HTTP/3 handling in production mode. + +## Reliability & Resilience + +- [x] Mutex-protected per-host context cache prevents concurrent generation races. +- [ ] Add bounded concurrency / connection limits. +- [ ] Add timeout and retry policy for upstream requests and TLS handshakes. +- [ ] Add graceful shutdown behavior for active MITM sessions under load. +- [ ] Add robust error taxonomy and user-facing failure modes. + +## Observability + +- [x] Debug output is gated behind `MITMConfig#debug`. +- [ ] Replace ad-hoc debug prints with structured logs for production diagnostics. +- [ ] Add metrics (handshake failures, generation latency, cache hits/misses, upstream status rates). +- [ ] Add health/readiness check endpoint or command. + +## Testing + +- [ ] Unit tests for certificate generator (key generation, CSR, signing, SAN DNS/IP). +- [ ] Integration tests for browser trust flow (Firefox/Chromium with local CA). +- [ ] Negative tests (invalid CA files, malformed CONNECT targets, broken cert cache files). +- [ ] Load/stress tests for concurrent CONNECT traffic and cert generation. + +## Operations + +- [~] Runtime sample exists (`samples/server_mitm.cr`) and README usage is documented. +- [ ] Add deployment profile (systemd/service config, restart policy, file permissions). +- [ ] Add operational runbook (bootstrap CA, trust install, rotation, incident recovery). +- [ ] Add secure secret management guidance for CA private key. + +## Current Assessment + +- Overall maturity: **Advanced MVP / Beta** +- Not yet production-ready due to outstanding hardening, testing depth, and operational controls. diff --git a/README.md b/README.md index 94b4368..f8c4ecf 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,60 @@ server = HTTP::Proxy::Server.new(handlers: [ end ``` +#### HTTPS MITM (MVP, CONNECT HTTP/1.1 only) + +```crystal +server = HTTP::Proxy::Server.new +server.mitm = HTTP::Proxy::Server::MITMConfig.new( + ca_certificate_path: "./certs/mitm-ca.crt", + ca_private_key_path: "./certs/mitm-ca.key", + certificate_cache_dir: "./.mitm-certs", + debug: false, +) + +address = server.bind_tcp("127.0.0.1", 8080) +puts "Listening on http://#{address}" +server.listen +``` + +Notes: + +- This enables HTTPS interception for `CONNECT` requests. +- The certificate authority (CA) used to sign MITM certs must be trusted by clients. +- This MVP is intended for HTTP/1.1 traffic over CONNECT. + +Certificate files: + +- `mitm-ca.crt` / `mitm-ca.key`: local CA (trust anchor used to sign dynamic leaf certs). +- `./.mitm-certs/*.crt` / `./.mitm-certs/*.key`: generated per-host leaf certificates. + +Static mode is still available: + +- `mitm.crt` / `mitm.key`: a single leaf certificate and key presented by the proxy. + +Firefox setup: + +- Import `mitm-ca.crt` in **Authorities** and trust it for websites. +- Do **not** import `mitm.crt` into Authorities. + +Server setup: + +- Dynamic mode (recommended): pass `mitm-ca.crt` as `ca_certificate_path` and `mitm-ca.key` as `ca_private_key_path`. +- Static mode: pass `mitm.crt` as `certificate_chain_path` and `mitm.key` as `private_key_path`. + +Run dynamic MITM sample: + +```bash +crystal run samples/server_mitm.cr +``` + +Useful flags: + +- `--ca-cert ./mitm-ca.crt` +- `--ca-key ./mitm-ca.key` +- `--cache-dir ./.mitm-certs` +- `--debug` (enable verbose MITM request/response output) + ### Client #### Make request with proxy @@ -112,7 +166,7 @@ puts response.body * [x] HTTPS Proxy: CONNECT support * [x] Make context.request & context.response writable * [x] Basic Authentication -* [ ] MITM HTTPS Proxy +* [x] MITM HTTPS Proxy (MVP, CONNECT HTTP/1.1) ### Proxy client diff --git a/mitm-ca.crt b/mitm-ca.crt new file mode 100644 index 0000000..aa1b2e0 --- /dev/null +++ b/mitm-ca.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFITCCAwmgAwIBAgIUKE5Bbf3Q2Y6N9QR1x4yEZPa0pFIwDQYJKoZIhvcNAQEL +BQAwGDEWMBQGA1UEAwwNTG9jYWwgTUlUTSBDQTAeFw0yNjAyMjcxNTM5MjhaFw0z +NjAyMjUxNTM5MjhaMBgxFjAUBgNVBAMMDUxvY2FsIE1JVE0gQ0EwggIiMA0GCSqG +SIb3DQEBAQUAA4ICDwAwggIKAoICAQDb8uaLhVtvrSso3j/G0HJBiz5zMl6zUBnz +zLuv0Dhqpyia4t/6qKu8jco4+iW978bjx4LvsNPzM/jq6Mc0I/BTH6opP/ckPGfL +IK7VYPoaWbQMFy0NpEmesgZgyb62TiSCuWtlBNDRYOyqriTcH4xMk9M1PKtx0GBp +cmphX+b0o/VHzXJ3WcxNnLn3k/vTmSGjzBV3eHE47A2iZVR0ukUm/GJOz3i+zaWq +CimY7VGUEuOK+iSf/zcXE+NcUkofdw3ZuLsRrBZZyQeh+lGRD7YU6OYsczuzb3XJ +AECIte83Ko03KkLxyoD+rYlT+CVb0S4O0w5TycGxLa5RzzCXmu0ZVoEtz9bw6VoD +eiBiI+FGlNFxgSi6WtRCsZ11Uw9Z/Pry2Ot62k18U9rS5lQhTOqplfEmksKIEsTK +7RDABEPeP8PPASrVr6BuSzgWq2FbVVu41tZ/TWwXwY9KsbhYTe9Sr8HbvWoT8mQS +UAe/lZWvCZvP8vp0fnHIj5Ci+2nGLy42p0Ft6ON7GEfbqQV63bKWZjP6jtgF7SNs +Tb7cMCyam4J5f181Uds0qMw0IdobUmG2LN64CNAVjF+8d+VRobkO8sKVYxIAAO+G +JeOTOtRSMvLD2+SQQ27XQBx4lMNeOo6fFs3jkcG7LQ8okXwo4lc9StVKWexh1WhP +APVBZs9DLQIDAQABo2MwYTAdBgNVHQ4EFgQUr4xEWjgXxTQgiArF09zoj4GIxw4w +HwYDVR0jBBgwFoAUr4xEWjgXxTQgiArF09zoj4GIxw4wDwYDVR0TAQH/BAUwAwEB +/zAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIBAMfKybOvRZalGj8p +IZxm/QERRYbk8mTuc9j/KZ4B6Nrr7DLKi5Dsx/dqBRYkdsQEJzxDw2RESYW+HxlS +BtLTpJyO4Mj9oBzaN0MFfKk8gAgNEeT7hbdKwiCoug26r/+LOfnhddlw1gayB8JF +Egu4x67EGWwP2gZJRJYY1/fjr8F39TByqhBWTzbmpq6INNu9hIfBSq+libPwnTNL +dBvmeY9mo7441HScXeGW8JJnf1sg3vGwC+3qDVFZilvV33/FRehb3g78yR8VTyp7 +9dEi0y/PqFOLOQFvvgwKo6g8yruoCPGeMLW99za6Xts79/tpj1QYIqsJb31ESxEc +ZBEqPXuRJOPjs2mGw3bnoYYlTrKB50gxezN1nOd4uvXohqKmoZ/S4Oc6fI8VuaHc +SfC9PJQ9AeW/DMgvBdo3dIc2KTd0536iW6RwybMKg7KGM7J39LRjbIriM1hU9wzV +KJW5wsUqENu7S+Vy2IUk8uZ4YJGPAbVbOSV2mpN03DJebpVeY6u5IoIj1LqDPZ/L +SXxKrlPSMCkrHfUoJWSIFuXglVYQnS0vXk7NyFbtT4tkb+7QO7XprJA9DD+oLpSK +GRH53WVAZL8JbCA5EifM5Ff24qCFHgzfe5fvNqUfF3qy5CyRmxugtEUzmAkJJfo/ +Y8ZH/T0v0jhMXQgbLRMr2t/Gz/xT +-----END CERTIFICATE----- diff --git a/mitm-ca.key b/mitm-ca.key new file mode 100644 index 0000000..9dcebda --- /dev/null +++ b/mitm-ca.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDb8uaLhVtvrSso +3j/G0HJBiz5zMl6zUBnzzLuv0Dhqpyia4t/6qKu8jco4+iW978bjx4LvsNPzM/jq +6Mc0I/BTH6opP/ckPGfLIK7VYPoaWbQMFy0NpEmesgZgyb62TiSCuWtlBNDRYOyq +riTcH4xMk9M1PKtx0GBpcmphX+b0o/VHzXJ3WcxNnLn3k/vTmSGjzBV3eHE47A2i +ZVR0ukUm/GJOz3i+zaWqCimY7VGUEuOK+iSf/zcXE+NcUkofdw3ZuLsRrBZZyQeh ++lGRD7YU6OYsczuzb3XJAECIte83Ko03KkLxyoD+rYlT+CVb0S4O0w5TycGxLa5R +zzCXmu0ZVoEtz9bw6VoDeiBiI+FGlNFxgSi6WtRCsZ11Uw9Z/Pry2Ot62k18U9rS +5lQhTOqplfEmksKIEsTK7RDABEPeP8PPASrVr6BuSzgWq2FbVVu41tZ/TWwXwY9K +sbhYTe9Sr8HbvWoT8mQSUAe/lZWvCZvP8vp0fnHIj5Ci+2nGLy42p0Ft6ON7GEfb +qQV63bKWZjP6jtgF7SNsTb7cMCyam4J5f181Uds0qMw0IdobUmG2LN64CNAVjF+8 +d+VRobkO8sKVYxIAAO+GJeOTOtRSMvLD2+SQQ27XQBx4lMNeOo6fFs3jkcG7LQ8o +kXwo4lc9StVKWexh1WhPAPVBZs9DLQIDAQABAoICABHD+V4sm9mV3aYT6YfX/1qO +8jDg0ShfoHECSOinA1+N9+gmyhuXcyOsOjjG77R7QQ/V5hwRJtV+jaz/t1NdUcSN +CrQBQCeTn3iXP7fpeNoXA8V0O8Xdzrp8O6qmsPpNroJGkseaj3lSAFu67CxBehYX +XJhwuZJcV+U8gh4yXlfFRIMTs5qzTJ66OYUnVVBejoqJ6fP37QFBg8ppr9wrzXkc +Kp6eAG089BQbdQeup4ezzOBFWx73QM5i41rqJKWM+rfqxWVkhnujErRBIUR7xePD +eg/+EMTqDFP7arsAIv3MOJLLnZOlHS00/CIlqXLUnwlAf+hBpBz5CRr/hrfAIkVi +ym8hzyTE961OmMV6UjSmUOZD+mgsSj/2MMji0CtHLd//U5YMHDfXDcakpuvS6HU6 +ovAI9TdMJQI5nkCbWSwxD8cZwoVH3xVR6UFb7rvuiP69XjRbE3PPzi05ya+ejMzf +LLC6wyqIOICn98gxBMfyxGMe6WcWpPEJqf95BT/PkKUpCjrGKfurGRAWxNOGvCPz +FEXFnCS8fvmsjx51XJ0hRoQeDiXs0WG/XVSSjz6tNIt8VoET40LbHhwXO7GJDjLh +mc/nSBS0pYtnvJ/bY66FgMGkdiEyKMCFYeAYPL13AlUrGV16hQJ5fnISQj+UAyqx +W03stcbCXA0Ed5DSNY3LAoIBAQD3v8nMp8xWPGRmIUPWyUmVxR/HpMCS9Xc1fO9I +how6j+vL2KLAdiIwcG6wUZuZdsQWVpWqK+Iwq6k2wRPnLsY4SVUhGhdMlpj/T0KG +ksntgohPtk6AJoHhqnW0U9JjGpYdCUCMJXz3n21qK0dNsiF7b4XjYymaXFvSL3Ry +b42EdeWS6X8Y/RG+h1ZHg+lD2buhrijf+GGNyrxzQomk7amKRIDMoFuGoGT/9oTO +YyMuZlPjZC0/AeG9mkRY3+B0rIObaFv1cwc0JROUeNLDPeB914gJhX7Xoe96NcMY +ZSzrQ4YCacPmg8raq3XwU8iy+R/trJry+KQG89F6cDiLxierAoIBAQDjRhj34OQB +0SxaJAOsgGCYWco49o9sySQcEd3P3p3ErwPNFnQCCOBaCLE6y2REeW1dWyHNwpJg +TlwHlVNj3qMg/Y0fBpwtMd30FPgp7b5Lxzv4zSr18+7YwttTrF4/7SmtXXlj6DrV +MJefyfCWxG8iQ1CNEB98CHq2BuizWvOwSJ2aI9taFGk//Q9pcQPTwGYof681Dks8 +a17MZfqNOaWCIlzHPOGIhlkjc2TOh1QBS9kE+bx14eXUxRJ/amITQttjgYSBIq8o +tp3vNkywlFhCscyayosPSbeGuOl3+Z2gdAbyj5KOd7Tcj63bJsml7/yAV4Jmv7r+ +tFt0QmEBOQiHAoIBAAOE+vXoUFPNSdPVlyQe+eehxEDOy1mLGSVuX+vU1XsjfkMI +Ec/QHc44WqowjphQgpqaokenlfABEEdR2NmI5ZH5ILd2qmwRG51M3/IPdcTk/NC9 +E0JoyaGODVwBcNStlQJWlk8nXS4bWq+Oa9XjuOwK+ojvaLDjrP5AZFQX15fRIPDE +VmThe3YMcCJV8mNfXXX/hl8gJSqhfanZgSERqz3mmTnO8V3pO2YTd3GDIQXQuFJb +ovTpLu7FmUD19TdTGA+GHQBQoQKRoESUrtHNODoxbKJN/i5MA53l056uGURCUk4I +eJr2tlQC6Yr/dbNtLJHwyMa414OtxQULQRJjPz8CggEAWPYWTejctw1elAYm3f3+ +UYRMENIKQCXXmZkwvu4/yT5MeZnBXQ6Gaxed8AqvO9JgCbvjVnxD+aiSg3FjC+OY +7Q/yjmNy/InZfHI81YS3CUh6ZCBDIbUTGAvl+DGvTsyRlMfS/VVougxkPWq5XvqT +GdFJlX3rJQzYo6m+qn3+h1FVR4GjmfYFYMO4pahUPC3CjzWzqkvnDUZl/BIq6d7X +t0GmGWLuURdtit/fZKw6KKu8ziLHL0l2QjvFytQkga+Y2rFW4YlnEMOyvHD/wdq/ +VZPtJ+YCWsCbMwPsd0bg+W6RTZ7/Wf7nb7JZ9j+PjQGMT9xxMbD5DDwi1DYrbGQb +vwKCAQBeXbIka07oCqcjpGiSaNelYya/1OLg+1saqGC+rKlIpjRPHm0o+wxziHFf +gY4tYei+VNGiLySu57Xd5k1ADJ3deuK+KePfLKCB2ehCs5rinw8dblaD/miM9VkK +/qpbFmjr7FM9n3Kd+sUI7a1r9sHy0+iMm78QNyDEI8RlISUSgVlLNHyETfACZuPn +F0c2gIYWCKZbzFiuieKttwzY4jkeD6LMn7OgjQ51qGKr1FjMQYjO+dPZYNRcq8f2 ++s1l3LeoLsY0jJC+cKbCrJLdaReRGkNkqmHa7j67JCp07k1NGVTATiEMYSy6Yfzm +2XDQweSmPKqwgTMSvbabrXiI0Ekm +-----END PRIVATE KEY----- diff --git a/samples/server.cr b/samples/server.cr index e76d05a..de5de05 100644 --- a/samples/server.cr +++ b/samples/server.cr @@ -2,7 +2,7 @@ require "../src/http_proxy" require "option_parser" host = "127.0.0.1" -port = 8081 +port = 8080 OptionParser.parse do |opts| opts.on("-h HOST", "--host HOST", "define host to run server") do |opt| diff --git a/samples/server_mitm.cr b/samples/server_mitm.cr new file mode 100644 index 0000000..5cea19d --- /dev/null +++ b/samples/server_mitm.cr @@ -0,0 +1,65 @@ +require "../src/http_proxy" +require "option_parser" + +host = "127.0.0.1" +port = 8080 +ca_certificate_path = "./mitm-ca.crt" +ca_private_key_path = "./mitm-ca.key" +certificate_cache_dir = "./.mitm-certs" +upstream_insecure = false +debug = false + +OptionParser.parse do |opts| + opts.banner = "Usage: crystal run samples/server_mitm.cr -- [arguments]" + + opts.on("-h HOST", "--host HOST", "define host to run server") do |opt| + host = opt + end + + opts.on("-p PORT", "--port PORT", "define port to run server") do |opt| + port = opt.to_i + end + + opts.on("--ca-cert PATH", "path to MITM CA certificate PEM (dynamic per-host cert mode)") do |opt| + ca_certificate_path = opt + end + + opts.on("--ca-key PATH", "path to MITM CA private key PEM (dynamic per-host cert mode)") do |opt| + ca_private_key_path = opt + end + + opts.on("--cache-dir PATH", "directory to store generated host certificates") do |opt| + certificate_cache_dir = opt + end + + opts.on("--upstream-insecure", "disable upstream TLS verification") do + upstream_insecure = true + end + + opts.on("--debug", "enable verbose MITM request/response debug output") do + debug = true + end +end + +server = HTTP::Proxy::Server.new(handlers: [ + HTTP::LogHandler.new, +]) + +server.mitm = HTTP::Proxy::Server::MITMConfig.new( + ca_certificate_path: ca_certificate_path, + ca_private_key_path: ca_private_key_path, + certificate_cache_dir: certificate_cache_dir, + upstream_insecure: upstream_insecure, + debug: debug, +) + +address = server.bind_tcp(host, port) +puts "Listening on http://#{address}" +puts "MITM mode enabled (CONNECT HTTP/1.1 MVP)" +puts "Certificate mode: dynamic per-host" +puts "CA certificate: #{ca_certificate_path}" +puts "CA private key: #{ca_private_key_path}" +puts "Certificate cache dir: #{certificate_cache_dir}" +puts "Upstream TLS verification: #{upstream_insecure ? "DISABLED" : "ENABLED"}" +puts "MITM debug output: #{debug ? "ENABLED" : "DISABLED"}" +server.listen diff --git a/src/ext/openssl/lib_crypro.cr b/src/ext/openssl/lib_crypro.cr new file mode 100644 index 0000000..f44d030 --- /dev/null +++ b/src/ext/openssl/lib_crypro.cr @@ -0,0 +1,48 @@ +require "openssl/lib_crypto" + +lib LibCrypto + alias EVP_PKEY = Void* + alias EVP_PKEY_CTX = Void* + alias X509_REQ = Void* + alias ASN1_INTEGER = Void* + alias ASN1_TIME = Void* + + EVP_PKEY_RSA = 6 + + fun evp_pkey_ctx_new_id = EVP_PKEY_CTX_new_id(id : Int32, e : Void*) : EVP_PKEY_CTX + fun evp_pkey_ctx_free = EVP_PKEY_CTX_free(ctx : EVP_PKEY_CTX) + fun evp_pkey_keygen_init = EVP_PKEY_keygen_init(ctx : EVP_PKEY_CTX) : Int32 + fun evp_pkey_ctx_ctrl_str = EVP_PKEY_CTX_ctrl_str(ctx : EVP_PKEY_CTX, type : UInt8*, value : UInt8*) : Int32 + fun evp_pkey_keygen = EVP_PKEY_keygen(ctx : EVP_PKEY_CTX, ppkey : EVP_PKEY*) : Int32 + fun evp_pkey_free = EVP_PKEY_free(pkey : EVP_PKEY) + + fun bio_new_file = BIO_new_file(filename : UInt8*, mode : UInt8*) : Bio* + fun bio_free_all = BIO_free_all(bio : Bio*) : Int32 + + fun pem_read_bio_private_key = PEM_read_bio_PrivateKey(bp : Bio*, x : EVP_PKEY*, cb : Void*, u : Void*) : EVP_PKEY + fun pem_write_bio_private_key = PEM_write_bio_PrivateKey(bp : Bio*, x : EVP_PKEY, enc : Void*, kstr : UInt8*, klen : Int32, cb : Void*, u : Void*) : Int32 + + fun pem_read_bio_x509 = PEM_read_bio_X509(bp : Bio*, x : X509*, cb : Void*, u : Void*) : X509 + fun pem_write_bio_x509_req = PEM_write_bio_X509_REQ(bp : Bio*, x : X509_REQ) : Int32 + fun pem_write_bio_x509 = PEM_write_bio_X509(bp : Bio*, x : X509) : Int32 + + fun x509_req_new = X509_REQ_new : X509_REQ + fun x509_req_free = X509_REQ_free(req : X509_REQ) + fun x509_req_set_version = X509_REQ_set_version(req : X509_REQ, version : Long) : Int32 + fun x509_req_set_subject_name = X509_REQ_set_subject_name(req : X509_REQ, name : X509_NAME) : Int32 + fun x509_req_set_pubkey = X509_REQ_set_pubkey(req : X509_REQ, pkey : EVP_PKEY) : Int32 + fun x509_req_sign = X509_REQ_sign(req : X509_REQ, pkey : EVP_PKEY, md : EVP_MD) : Int32 + fun x509_req_get_subject_name = X509_REQ_get_subject_name(req : X509_REQ) : X509_NAME + fun x509_req_get_pubkey = X509_REQ_get_pubkey(req : X509_REQ) : EVP_PKEY + + fun x509_set_version = X509_set_version(x : X509, version : Long) : Int32 + fun x509_set_issuer_name = X509_set_issuer_name(x : X509, name : X509_NAME) : Int32 + fun x509_set_pubkey = X509_set_pubkey(x : X509, pkey : EVP_PKEY) : Int32 + fun x509_sign = X509_sign(x : X509, pkey : EVP_PKEY, md : EVP_MD) : Int32 + fun x509_get_serial_number = X509_get_serialNumber(x : X509) : ASN1_INTEGER + fun x509_getm_not_before = X509_getm_notBefore(x : X509) : ASN1_TIME + fun x509_getm_not_after = X509_getm_notAfter(x : X509) : ASN1_TIME + + fun asn1_integer_set = ASN1_INTEGER_set(a : ASN1_INTEGER, v : Long) : Int32 + fun x509_gmtime_adj = X509_gmtime_adj(s : ASN1_TIME, adj : Long) : ASN1_TIME +end diff --git a/src/http/proxy/server.cr b/src/http/proxy/server.cr index 6cd5ca0..4253004 100644 --- a/src/http/proxy/server.cr +++ b/src/http/proxy/server.cr @@ -2,6 +2,7 @@ require "socket" require "wait_group" require "./server/handler" require "./server/context" +require "./server/mitm_config" {% if !flag?(:without_openssl) %} require "openssl" {% end %} @@ -28,6 +29,8 @@ class HTTP::Proxy::Server # Returns `true` if this server is listening on its sockets. getter? listening : Bool = false + property mitm : MITMConfig? + # Creates a new HTTP Proxy server def initialize handler = build_middleware @@ -61,7 +64,7 @@ class HTTP::Proxy::Server end private def build_middleware(handler : (Context ->)? = nil) - proxy_handler = Handler.new + proxy_handler = Handler.new(-> { @mitm }) proxy_handler.next = handler if handler proxy_handler end diff --git a/src/http/proxy/server/certificate_generator.cr b/src/http/proxy/server/certificate_generator.cr new file mode 100644 index 0000000..7a97bd7 --- /dev/null +++ b/src/http/proxy/server/certificate_generator.cr @@ -0,0 +1,211 @@ +require "socket" +require "./../../../ext/openssl/lib_crypro.cr" + +class HTTP::Proxy::Server + class CertificateGenerator + # Native equivalent of: + # openssl genrsa -out + def generate_private_key(path : String, bits : Int32 = 2048) : Bool + ctx = LibCrypto.evp_pkey_ctx_new_id(LibCrypto::EVP_PKEY_RSA, Pointer(Void).null) + return false if ctx.null? + + pkey = Pointer(LibCrypto::EVP_PKEY).malloc(1, Pointer(Void).null) + + begin + return false if LibCrypto.evp_pkey_keygen_init(ctx) <= 0 + return false if LibCrypto.evp_pkey_ctx_ctrl_str(ctx, "rsa_keygen_bits".to_unsafe, bits.to_s.to_unsafe) <= 0 + return false if LibCrypto.evp_pkey_keygen(ctx, pkey) <= 0 + return false if pkey.value.null? + + bio = LibCrypto.bio_new_file(path.to_unsafe, "w".to_unsafe) + return false if bio.null? + + begin + return false if LibCrypto.pem_write_bio_private_key(bio, pkey.value, Pointer(Void).null, Pointer(UInt8).null, 0, Pointer(Void).null, Pointer(Void).null) != 1 + ensure + LibCrypto.bio_free_all(bio) + end + + true + ensure + LibCrypto.evp_pkey_free(pkey.value) unless pkey.value.null? + LibCrypto.evp_pkey_ctx_free(ctx) + end + end + + # Native equivalent of the combined CLI flow: + # openssl genrsa -out 2048 + # openssl req -new -key -out -subj "/CN=" + # openssl x509 -req -in -CA -CAkey \ + # -out -days 825 -sha256 -extfile + def generate(*, + host : String, + cert_path : String, + key_path : String, + ca_cert_path : String, + ca_key_path : String, + serial_path : String) : Bool + return false unless generate_private_key(key_path) + + serial = next_serial(serial_path) + + host_key = load_private_key(key_path) + return false if host_key.null? + + ca_cert = load_certificate(ca_cert_path) + return false if ca_cert.null? + + ca_key = load_private_key(ca_key_path) + return false if ca_key.null? + + csr = create_csr(host, host_key) + return false if csr.null? + + cert = sign_csr(host, serial, csr, ca_cert, ca_key) + return false if cert.null? + + return false unless write_certificate(cert_path, cert) + true + ensure + LibCrypto.x509_req_free(csr) if csr && !csr.null? + LibCrypto.x509_free(cert) if cert && !cert.null? + LibCrypto.evp_pkey_free(host_key) if host_key && !host_key.null? + LibCrypto.x509_free(ca_cert) if ca_cert && !ca_cert.null? + LibCrypto.evp_pkey_free(ca_key) if ca_key && !ca_key.null? + end + + # Native equivalent of: + # openssl pkey -in + private def load_private_key(path : String) : LibCrypto::EVP_PKEY + bio = LibCrypto.bio_new_file(path.to_unsafe, "r".to_unsafe) + return Pointer(Void).null.as(LibCrypto::EVP_PKEY) if bio.null? + + begin + key = LibCrypto.pem_read_bio_private_key(bio, Pointer(LibCrypto::EVP_PKEY).null, Pointer(Void).null, Pointer(Void).null) + key || Pointer(Void).null.as(LibCrypto::EVP_PKEY) + ensure + LibCrypto.bio_free_all(bio) + end + end + + # Native equivalent of: + # openssl x509 -in + private def load_certificate(path : String) : LibCrypto::X509 + bio = LibCrypto.bio_new_file(path.to_unsafe, "r".to_unsafe) + return Pointer(Void).null.as(LibCrypto::X509) if bio.null? + + begin + cert = LibCrypto.pem_read_bio_x509(bio, Pointer(LibCrypto::X509).null, Pointer(Void).null, Pointer(Void).null) + cert || Pointer(Void).null.as(LibCrypto::X509) + ensure + LibCrypto.bio_free_all(bio) + end + end + + # Native equivalent of: + # openssl req -new -key -subj "/CN=" + private def create_csr(host : String, host_key : LibCrypto::EVP_PKEY) : LibCrypto::X509_REQ + req = LibCrypto.x509_req_new + return Pointer(Void).null.as(LibCrypto::X509_REQ) if req.null? + + subject = LibCrypto.x509_name_new + return Pointer(Void).null.as(LibCrypto::X509_REQ) if subject.null? + + begin + return Pointer(Void).null.as(LibCrypto::X509_REQ) if LibCrypto.x509_name_add_entry_by_txt(subject, "CN".to_unsafe, LibCrypto::MBSTRING_UTF8, host.to_unsafe, -1, -1, 0).null? + return Pointer(Void).null.as(LibCrypto::X509_REQ) if LibCrypto.x509_req_set_version(req, 0) != 1 + return Pointer(Void).null.as(LibCrypto::X509_REQ) if LibCrypto.x509_req_set_subject_name(req, subject) != 1 + return Pointer(Void).null.as(LibCrypto::X509_REQ) if LibCrypto.x509_req_set_pubkey(req, host_key) != 1 + return Pointer(Void).null.as(LibCrypto::X509_REQ) if LibCrypto.x509_req_sign(req, host_key, LibCrypto.evp_sha256) <= 0 + ensure + LibCrypto.x509_name_free(subject) + end + + req + end + + # Native equivalent of: + # openssl x509 -req -in -CA -CAkey \ + # -days 825 -sha256 -extfile + private def sign_csr(host : String, serial : Int64, req : LibCrypto::X509_REQ, ca_cert : LibCrypto::X509, ca_key : LibCrypto::EVP_PKEY) : LibCrypto::X509 + cert = LibCrypto.x509_new + return Pointer(Void).null.as(LibCrypto::X509) if cert.null? + + req_pubkey = LibCrypto.x509_req_get_pubkey(req) + return Pointer(Void).null.as(LibCrypto::X509) if req_pubkey.null? + + begin + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_set_version(cert, 2) != 1 + serial_number = LibCrypto.x509_get_serial_number(cert) + return Pointer(Void).null.as(LibCrypto::X509) if serial_number.null? + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.asn1_integer_set(serial_number, serial) != 1 + + not_before = LibCrypto.x509_getm_not_before(cert) + not_after = LibCrypto.x509_getm_not_after(cert) + return Pointer(Void).null.as(LibCrypto::X509) if not_before.null? || not_after.null? + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_gmtime_adj(not_before, 0).null? + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_gmtime_adj(not_after, 825_i64 * 24 * 60 * 60).null? + + req_subject = LibCrypto.x509_req_get_subject_name(req) + issuer_subject = LibCrypto.x509_get_subject_name(ca_cert) + return Pointer(Void).null.as(LibCrypto::X509) if req_subject.null? || issuer_subject.null? + + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_set_subject_name(cert, req_subject) != 1 + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_set_issuer_name(cert, issuer_subject) != 1 + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_set_pubkey(cert, req_pubkey) != 1 + + return Pointer(Void).null.as(LibCrypto::X509) unless add_extension(cert, "basicConstraints", "critical,CA:FALSE") + return Pointer(Void).null.as(LibCrypto::X509) unless add_extension(cert, "keyUsage", "critical,digitalSignature,keyEncipherment") + return Pointer(Void).null.as(LibCrypto::X509) unless add_extension(cert, "extendedKeyUsage", "serverAuth") + + san_value = Socket::IPAddress.valid?(host) ? "IP:#{host}" : "DNS:#{host}" + return Pointer(Void).null.as(LibCrypto::X509) unless add_extension(cert, "subjectAltName", san_value) + + return Pointer(Void).null.as(LibCrypto::X509) if LibCrypto.x509_sign(cert, ca_key, LibCrypto.evp_sha256) <= 0 + + cert + ensure + LibCrypto.evp_pkey_free(req_pubkey) + end + end + + private def next_serial(serial_path : String) : Int64 + current = if File.exists?(serial_path) + File.read(serial_path).strip.to_i64? + end + base = current || (Time.utc.to_unix * 1000) + serial = base + 1 + File.write(serial_path, serial.to_s) + serial + end + + # Native equivalent of adding extension entries via extfile in `openssl x509 -extfile ...`. + private def add_extension(cert : LibCrypto::X509, name : String, value : String) : Bool + nid = LibCrypto.obj_sn2nid(name.to_unsafe) + nid = LibCrypto.obj_ln2nid(name.to_unsafe) if nid == LibCrypto::NID_undef + return false if nid == LibCrypto::NID_undef + + ext = LibCrypto.x509v3_ext_nconf_nid(Pointer(Void).null, Pointer(Void).null, nid, value.to_unsafe) + return false if ext.null? + + begin + !LibCrypto.x509_add_ext(cert, ext, -1).null? + ensure + LibCrypto.x509_extension_free(ext) + end + end + + # Native equivalent of: + # openssl x509 -out + private def write_certificate(path : String, cert : LibCrypto::X509) : Bool + bio = LibCrypto.bio_new_file(path.to_unsafe, "w".to_unsafe) + return false if bio.null? + + begin + LibCrypto.pem_write_bio_x509(bio, cert) == 1 + ensure + LibCrypto.bio_free_all(bio) + end + end + end +end diff --git a/src/http/proxy/server/context.cr b/src/http/proxy/server/context.cr index 96a643b..0cbf3a4 100644 --- a/src/http/proxy/server/context.cr +++ b/src/http/proxy/server/context.cr @@ -1,3 +1,7 @@ +{% if !flag?(:without_openssl) %} + require "openssl" +{% end %} + class HTTP::Proxy::Server class Context # The `HTTP::Request` to process. @@ -7,7 +11,7 @@ class HTTP::Proxy::Server getter response : HTTP::Server::Response # :nodoc: - def initialize(@request : HTTP::Request, @response : HTTP::Server::Response) + def initialize(@request : HTTP::Request, @response : HTTP::Server::Response, @mitm : HTTP::Proxy::Server::MITMConfig? = nil) end def perform @@ -15,14 +19,22 @@ class HTTP::Proxy::Server when "OPTIONS" @response.headers["Allow"] = "OPTIONS,GET,HEAD,POST,PUT,DELETE,CONNECT" when "CONNECT" - handle_tunneling + {% unless flag?(:without_openssl) %} + if mitm = @mitm + handle_tunneling_mitm(mitm) + else + handle_tunneling + end + {% else %} + handle_tunneling + {% end %} else handle_http end end private def handle_tunneling - host, port = @request.resource.split(":", 2) + host, port = connect_target upstream = TCPSocket.new(host, port) @response.upgrade do |downstream| @@ -36,12 +48,184 @@ class HTTP::Proxy::Server end end + {% unless flag?(:without_openssl) %} + private def handle_tunneling_mitm(mitm : HTTP::Proxy::Server::MITMConfig) + host, port = connect_target + + if mitm.bypass_target?(host, port) + debug_puts(mitm, "MITM bypass cache hit for #{host}:#{port}") + handle_tunneling + return + end + + @response.upgrade do |downstream| + downstream = downstream.as(TCPSocket) + downstream.sync = true + + tls_downstream = begin + OpenSSL::SSL::Socket::Server.new(downstream, context: mitm.server_context_for(host), sync_close: true) + rescue ex : OpenSSL::SSL::Error + debug_puts(mitm, "MITM TLS handshake failed for #{host}:#{port} - #{ex.message}") + next + end + tls_downstream.sync = true + debug_puts(mitm, "MITM TLS established for #{host}:#{port}, ALPN=#{tls_downstream.alpn_protocol || "none"}") + + upstream_tls_context = mitm.upstream_context || OpenSSL::SSL::Context::Client.new + upstream_tls_context.alpn_protocol = "http/1.1" + upstream_tcp = TCPSocket.new(host, port) + upstream_tcp.sync = true + upstream_tls = OpenSSL::SSL::Socket::Client.new(upstream_tcp, context: upstream_tls_context, sync_close: true, hostname: host.rchop('.')) + upstream_tls.sync = true + debug_puts(mitm, "MITM upstream TLS established for #{host}:#{port}, ALPN=#{upstream_tls.alpn_protocol || "none"}") + + begin + request_count = 0 + + loop do + request_count += 1 + debug_puts(mitm, "MITM waiting downstream request ##{request_count} for #{host}:#{port}") + parsed = HTTP::Request.from_io(tls_downstream) + + case parsed + when Nil + debug_puts(mitm, "MITM downstream EOF for #{host}:#{port}") + break + when HTTP::Status + debug_puts(mitm, "MITM downstream request parsing failed for #{host}:#{port} (non-HTTP/1.x stream?)") + break + when HTTP::Request + debug_puts(mitm, "MITM request ##{request_count}: #{parsed.method} #{parsed.resource} #{parsed.version} host=#{parsed.headers["Host"]? || "nil"}") + + if websocket_upgrade_request?(parsed) + parsed.headers.delete("Proxy-Authorization") + parsed.headers.delete("Proxy-Connection") + + parsed.to_io(upstream_tls) + upstream_tls.flush + + mitm.mark_bypass_target(host, port) + debug_puts(mitm, "MITM detected websocket upgrade for #{host}:#{port}; bypassing via raw tunnel") + + WaitGroup.wait do |wg| + wg.spawn { transfer(upstream_tls, tls_downstream) } + wg.spawn { transfer(tls_downstream, upstream_tls) } + end + break + end + + if parsed.method.in?({"POST", "PUT", "PATCH"}) + request_body = parsed.body.try(&.gets_to_end) || "" + debug_puts(mitm, "MITM request ##{request_count} body bytes=#{request_body.bytesize}") + debug_puts(mitm, "MITM request ##{request_count} body BEGIN") + debug_puts(mitm, request_body) + debug_puts(mitm, "MITM request ##{request_count} body END") + + content_type = parsed.headers["Content-Type"]? + if content_type && content_type.starts_with?("application/x-www-form-urlencoded") + begin + params = URI::Params.parse(request_body) + debug_puts(mitm, "MITM request ##{request_count} form params: #{params}") + rescue ex + debug_puts(mitm, "MITM request ##{request_count} form params parse failed: #{ex.message}") + end + end + + parsed.body = request_body + end + + parsed.headers.delete("Proxy-Authorization") + parsed.headers.delete("Proxy-Connection") + parsed.headers["Accept-Encoding"] = "identity" + + parsed.to_io(upstream_tls) + upstream_tls.flush + + response = begin + HTTP::Client::Response.from_io(upstream_tls, ignore_body: parsed.ignore_body?, decompress: false) + rescue ex + debug_puts(mitm, "MITM upstream response parse failed for #{host}:#{port} request ##{request_count}: #{ex.class}: #{ex.message}") + mitm.mark_bypass_target(host, port) + debug_puts(mitm, "MITM marked bypass target #{host}:#{port} due to incompatible upstream protocol") + break + end + debug_puts(mitm, "MITM response ##{request_count}: status=#{response.status_code} keep_alive=#{response.keep_alive?} content_length=#{response.headers["Content-Length"]? || "nil"} transfer_encoding=#{response.headers["Transfer-Encoding"]? || "nil"} content_type=#{response.headers["Content-Type"]? || "nil"}") + + if response.status_code == 101 + response.to_io(tls_downstream) + tls_downstream.flush + debug_puts(mitm, "MITM response ##{request_count}: upgrade 101 detected, switching to raw tunnel") + + WaitGroup.wait do |wg| + wg.spawn { transfer(upstream_tls, tls_downstream) } + wg.spawn { transfer(tls_downstream, upstream_tls) } + end + break + end + + response.consume_body_io + body_size = response.body.bytesize + debug_puts(mitm, "MITM response ##{request_count}: buffered body bytes=#{body_size}") + # debug_puts(mitm, "MITM response ##{request_count} body BEGIN") + # debug_puts(mitm, response.body) + # debug_puts(mitm, "MITM response ##{request_count} body END") + + response.headers.delete("Transfer-Encoding") + response.headers["Content-Length"] = body_size.to_s + + response.to_io(tls_downstream) + tls_downstream.flush + debug_puts(mitm, "MITM response ##{request_count} forwarded and flushed") + + keep_alive = parsed.keep_alive? && response.keep_alive? + debug_puts(mitm, "MITM keep-alive ##{request_count}: request=#{parsed.keep_alive?} response=#{response.keep_alive?} -> #{keep_alive}") + break unless keep_alive + end + end + ensure + upstream_tls.close + end + end + rescue ex + Log.error(exception: ex) { "Unhandled exception on HTTP::Proxy::Server::Context MITM tunnel" } + end + + private def debug_puts(mitm : HTTP::Proxy::Server::MITMConfig, message : String) + puts message if mitm.debug? + end + {% end %} + + private def connect_target : {String, Int32} + resource = @request.resource + separator = resource.rindex(':') + raise IO::Error.new("Invalid CONNECT target: #{resource}") unless separator + + host = resource[0, separator] + host = host[1..-2] if host.starts_with?('[') && host.ends_with?(']') + port = resource[(separator + 1)..].to_i + {host, port} + rescue ex + raise IO::Error.new("Invalid CONNECT target: #{resource}", cause: ex) + end + private def transfer(destination, source) - IO.copy(destination, source) + IO.copy(source, destination) rescue ex Log.error(exception: ex) { "Unhandled exception on HTTP::Proxy::Server::Context" } end + private def websocket_upgrade_request?(request : HTTP::Request) : Bool + upgrade = request.headers["Upgrade"]? + return false unless upgrade && upgrade.compare("websocket", case_insensitive: true) == 0 + + connection = request.headers["Connection"]? + return false unless connection + + connection.split(',').any? do |token| + token.strip.compare("upgrade", case_insensitive: true) == 0 + end + end + private def handle_http host = @request.hostname || "" client = HTTP::Client.new(host) diff --git a/src/http/proxy/server/handler.cr b/src/http/proxy/server/handler.cr index 43603b1..473c348 100644 --- a/src/http/proxy/server/handler.cr +++ b/src/http/proxy/server/handler.cr @@ -3,10 +3,13 @@ require "./context" class HTTP::Proxy::Server::Handler include HTTP::Handler + def initialize(@mitm_config_provider : -> HTTP::Proxy::Server::MITMConfig?) + end + property next : HTTP::Handler | HTTP::Handler::HandlerProc | HandlerProc? def call(context) - HTTP::Proxy::Server::Context.new(context.request, context.response).perform + HTTP::Proxy::Server::Context.new(context.request, context.response, @mitm_config_provider.call).perform end alias HandlerProc = HTTP::Proxy::Server::Context -> diff --git a/src/http/proxy/server/mitm_config.cr b/src/http/proxy/server/mitm_config.cr new file mode 100644 index 0000000..83e49cf --- /dev/null +++ b/src/http/proxy/server/mitm_config.cr @@ -0,0 +1,131 @@ +require "./certificate_generator" +require "set" + +class HTTP::Proxy::Server + class MITMConfig + getter certificate_chain_path : String? + getter private_key_path : String? + getter ca_certificate_path : String? + getter ca_private_key_path : String? + getter certificate_cache_dir : String + getter? upstream_insecure : Bool + getter? debug : Bool + + def initialize(@certificate_chain_path : String, @private_key_path : String, @upstream_insecure : Bool = false, @debug : Bool = false) + @ca_certificate_path = nil + @ca_private_key_path = nil + @certificate_cache_dir = ".mitm-certs" + end + + def initialize(*, @ca_certificate_path : String, @ca_private_key_path : String, + @certificate_cache_dir : String = ".mitm-certs", @upstream_insecure : Bool = false, + @debug : Bool = false) + @certificate_chain_path = nil + @private_key_path = nil + end + + {% unless flag?(:without_openssl) %} + @server_context : OpenSSL::SSL::Context::Server? + @server_context_by_host = {} of String => OpenSSL::SSL::Context::Server + @bypass_targets = Set(String).new + @mutex = Mutex.new + @certificate_generator = CertificateGenerator.new + + def bypass_target?(host : String, port : Int32) : Bool + @mutex.synchronize do + @bypass_targets.includes?("#{host}:#{port}") + end + end + + def mark_bypass_target(host : String, port : Int32) : Nil + @mutex.synchronize do + @bypass_targets.add("#{host}:#{port}") + end + end + + def server_context : OpenSSL::SSL::Context::Server + @server_context ||= begin + cert_path = @certificate_chain_path + key_path = @private_key_path + raise ArgumentError.new("Static MITM mode requires certificate_chain_path and private_key_path") unless cert_path && key_path + + context = OpenSSL::SSL::Context::Server.new + context.certificate_chain = cert_path + context.private_key = key_path + context.alpn_protocol = "http/1.1" + context + end + end + + def server_context_for(host : String) : OpenSSL::SSL::Context::Server + return server_context unless dynamic_certificates? + + if context = @server_context_by_host[host]? + return context + end + + @mutex.synchronize do + if context = @server_context_by_host[host]? + return context + end + + cert_path, key_path = ensure_host_certificate(host) + + context = OpenSSL::SSL::Context::Server.new + context.certificate_chain = cert_path + context.private_key = key_path + context.alpn_protocol = "http/1.1" + + @server_context_by_host[host] = context + context + end + end + + private def dynamic_certificates? : Bool + !!(@ca_certificate_path && @ca_private_key_path) + end + + private def ensure_host_certificate(host : String) : {String, String} + host_key = sanitize_host_for_path(host) + cert_path = File.join(@certificate_cache_dir, "#{host_key}.crt") + key_path = File.join(@certificate_cache_dir, "#{host_key}.key") + + return {cert_path, key_path} if File.exists?(cert_path) && File.exists?(key_path) + + Dir.mkdir_p(@certificate_cache_dir) + create_host_certificate(host, cert_path, key_path, host_key) + {cert_path, key_path} + end + + private def sanitize_host_for_path(host : String) : String + host.gsub(/[^a-zA-Z0-9\.\-]/, "_") + end + + private def create_host_certificate(host : String, cert_path : String, key_path : String, host_key : String) : Nil + ca_cert_path = @ca_certificate_path + ca_key_path = @ca_private_key_path + raise ArgumentError.new("Dynamic MITM mode requires ca_certificate_path and ca_private_key_path") unless ca_cert_path && ca_key_path + + serial_path = File.join(@certificate_cache_dir, "ca.srl") + generated = @certificate_generator.generate( + host: host, + cert_path: cert_path, + key_path: key_path, + ca_cert_path: ca_cert_path, + ca_key_path: ca_key_path, + serial_path: serial_path, + ) + + raise IO::Error.new("Certificate generation failed for #{host}") unless generated + end + + def upstream_context : OpenSSL::SSL::Context::Client? + return nil unless @upstream_insecure + + context = OpenSSL::SSL::Context::Client.insecure + context.verify_mode = OpenSSL::SSL::VerifyMode::NONE + context + end + {% end %} + end +end