From cca983e10d3d49ac5a12d1f527a4408477b84fa8 Mon Sep 17 00:00:00 2001 From: Burke Libbey Date: Tue, 26 May 2026 16:50:03 -0400 Subject: [PATCH] add v3 X25519 + ML-KEM-768 scheme --- CHANGELOG.md | 6 + README.md | 56 ++++-- cmd/ejson/actions.go | 13 +- cmd/ejson/main.go | 15 +- crypto/boxed_message.go | 94 ++++++---- crypto/crypto.go | 16 +- crypto/crypto_test.go | 36 ++-- crypto/hybrid.go | 241 +++++++++++++++++++++++++ crypto/hybrid_test.go | 216 ++++++++++++++++++++++ crypto/keys.go | 389 ++++++++++++++++++++++++++++++++++++++++ dev.yml | 2 +- ejson.go | 67 ++++--- ejson_test.go | 67 +++++++ json/key.go | 71 +++++--- json/key_test.go | 17 ++ 15 files changed, 1159 insertions(+), 147 deletions(-) create mode 100644 crypto/hybrid.go create mode 100644 crypto/hybrid_test.go create mode 100644 crypto/keys.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 62fba6a..35eb45e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Unreleased + +* Add hybrid post-quantum `v3` encryption using X25519 + ML-KEM-768, HKDF-SHA256, and XChaCha20-Poly1305. +* Add `ejson keygen -scheme v3` / `-pqc` and schema-aware keydir lookup using v3 key IDs. +* Preserve legacy `v1` key generation and decryption compatibility. + # 1.5.4 * Bumps golang.org/x/crypto from 0.17.0 to 0.31.0 diff --git a/README.md b/README.md index dcdfbd3..965aedd 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,13 @@ # ejson `ejson` is a utility for managing a collection of secrets in source control. The -secrets are encrypted using [public -key](http://en.wikipedia.org/wiki/Public-key_cryptography), [elliptic -curve](http://en.wikipedia.org/wiki/Elliptic_curve_cryptography) cryptography -([NaCl](http://nacl.cr.yp.to/) [Box](http://nacl.cr.yp.to/box.html): -[Curve25519](http://en.wikipedia.org/wiki/Curve25519) + -[Salsa20](http://en.wikipedia.org/wiki/Salsa20) + -[Poly1305-AES](http://en.wikipedia.org/wiki/Poly1305-AES)). Secrets are -collected in a JSON file, in which all the string values are encrypted. Public -keys are embedded in the file, and the decrypter looks up the corresponding -private key from its local filesystem. +secrets are encrypted using public-key cryptography. Legacy `v1` keys use +[NaCl](http://nacl.cr.yp.to/) [Box](http://nacl.cr.yp.to/box.html) +(Curve25519 + XSalsa20 + Poly1305). New hybrid `v3` keys use X25519 and +ML-KEM-768 to derive a shared secret with HKDF-SHA256, then encrypt values with +XChaCha20-Poly1305. Secrets are collected in a JSON file, in which all the +string values are encrypted. Public keys are embedded in the file, and the +decrypter looks up the corresponding private key from its local filesystem. ![demo](http://burkelibbey.s3.amazonaws.com/ejson-demo.gif) @@ -80,6 +77,21 @@ $ cat /opt/ejson/keys/5339* 888a4291bef9135729357b8c70e5a62b0bbe104a679d829cdbe56d46a4481aaf ``` +By default, `keygen` still generates legacy `v1` keys. To generate a hybrid +post-quantum keypair, use `-scheme v3` or `-pqc`: + +``` +$ ejson keygen -scheme v3 -w +v3: +$ ls /opt/ejson/keys +<32-character key id> +``` + +Hybrid public keys are intentionally much longer than legacy keys because the +full ML-KEM public key is embedded in the EJSON document. This preserves the +existing workflow where anyone who can edit the document can add new secrets +without access to the private keydir. + ### 3: Create an `ejson` file The format is described in more detail [later on](#format). For now, create a @@ -115,10 +127,13 @@ new secret will be encrypted. ### 5: Decrypt the file -To decrypt the file, you must have a file present in the `keydir` whose name is -the 64-byte hex-encoded public key exactly as embedded in the `ejson` document. -The contents of that file must be the similarly-encoded private key. If you used -`ejson keygen -w`, you've already got this covered. +To decrypt the file, you must have the matching private key present in the +`keydir`. For legacy `v1` documents, the keydir filename is the 64-character +hex-encoded public key exactly as embedded in the `ejson` document, and the file +contains the similarly-encoded private key. For hybrid `v3` documents, the +keydir filename is a 32-character key ID derived from the public key, and the +file contains an `ejson-key v3` private key file with both X25519 and ML-KEM key +material. If you used `ejson keygen -w`, you've already got this covered. Unlike `ejson encrypt`, which overwrites the specified files, `ejson decrypt` only takes one file parameter, and prints the output to `stdout`: @@ -138,9 +153,11 @@ The `ejson` document format is simple, but there are a few points to be aware of: 1. It's just JSON. -2. There *must* be a key at the top level named `_public_key`, whose value is a - 32-byte hex-encoded (i.e. 64 ASCII byte) public key as generated by `ejson - keygen`. +2. There *must* be a key at the top level named `_public_key`. For legacy `v1` + documents, its value is a 32-byte hex-encoded (i.e. 64 ASCII byte) public key + as generated by `ejson keygen`. For hybrid `v3` documents, its value begins + with `v3:` and contains a base64-encoded X25519 public key plus ML-KEM-768 + encapsulation key. 3. Any string literal that isn't an object key will be encrypted by default (ie. in `{"a": "b"}`, `"b"` will be encrypted, but `"a"` will not. 4. Numbers, booleans, and nulls aren't encrypted. @@ -149,6 +166,11 @@ of: encrypted, and is useful for implementing metadata schemes. 6. Underscores do not propagate downward. For example, in `{"_a": {"b": "c"}}`, `"c"` will be encrypted. +7. Encrypted values are schema-tagged. Legacy values use + `EJ[1:::]`. Hybrid post-quantum + values use + `EJ[3::::]`. + Existing `EJ[1:...]` values remain decryptable. ## See also diff --git a/cmd/ejson/actions.go b/cmd/ejson/actions.go index 51558ff..19d3a6b 100644 --- a/cmd/ejson/actions.go +++ b/cmd/ejson/actions.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strings" "github.com/Shopify/ejson" ) @@ -43,15 +44,19 @@ func decryptAction(args []string, keydir, userSuppliedPrivateKey, outFile string return err } -func keygenAction(_ []string, keydir string, wFlag bool) error { - pub, priv, err := ejson.GenerateKeypair() +func keygenAction(_ []string, keydir string, wFlag bool, scheme string) error { + pub, priv, keyID, err := ejson.GenerateKeypairForScheme(scheme) if err != nil { return err } if wFlag { - keyFile := fmt.Sprintf("%s/%s", keydir, pub) - err := writeFile(keyFile, append([]byte(priv), '\n'), 0o440) + keyFile := fmt.Sprintf("%s/%s", keydir, keyID) + contents := []byte(priv) + if !strings.HasSuffix(priv, "\n") { + contents = append(contents, '\n') + } + err := writeFile(keyFile, contents, 0o440) if err != nil { return err } diff --git a/cmd/ejson/main.go b/cmd/ejson/main.go index 31253b6..5c18a4b 100644 --- a/cmd/ejson/main.go +++ b/cmd/ejson/main.go @@ -78,9 +78,22 @@ func main() { Name: "write, w", Usage: "rather than printing both keys, print the public and write the private into the keydir", }, + cli.StringFlag{ + Name: "scheme", + Value: "v1", + Usage: "key scheme to generate: v1 or v3", + }, + cli.BoolFlag{ + Name: "pqc", + Usage: "generate a v3 hybrid X25519+ML-KEM-768 keypair", + }, }, Action: func(c *cli.Context) { - if err := keygenAction(c.Args(), c.GlobalString("keydir"), c.Bool("write")); err != nil { + scheme := c.String("scheme") + if c.Bool("pqc") { + scheme = "v3" + } + if err := keygenAction(c.Args(), c.GlobalString("keydir"), c.Bool("write"), scheme); err != nil { fmt.Fprintln(os.Stderr, "Key generation failed:", err) os.Exit(1) } diff --git a/crypto/boxed_message.go b/crypto/boxed_message.go index 1799667..87f27ef 100644 --- a/crypto/boxed_message.go +++ b/crypto/boxed_message.go @@ -3,13 +3,16 @@ package crypto import ( "encoding/base64" "fmt" - "regexp" "strconv" + "strings" ) -var messageParser = regexp.MustCompile("\\AEJ\\[(\\d):([A-Za-z0-9+=/]{44}):([A-Za-z0-9+=/]{32}):(.+)\\]\\z") +const ( + boxedMessagePrefix = "EJ[" + boxedMessageSuffix = "]" +) -// boxedMessage dumps and loads the wire format for encrypted messages. The +// boxedMessage dumps and loads the v1 wire format for encrypted messages. The // schema is fairly simple: // // "EJ[" @@ -28,14 +31,26 @@ type boxedMessage struct { Box []byte } -// IsBoxedMessage tests whether a value is formatted using the boxedMessage -// format. This can be used to determine whether a string value requires +// IsBoxedMessage tests whether a value is formatted using a supported boxed +// message format. This can be used to determine whether a string value requires // encryption or is already encrypted. func IsBoxedMessage(data []byte) bool { - return messageParser.Find(data) != nil + version, fields, err := parseBoxedEnvelope(data) + if err != nil { + return false + } + + switch version { + case SchemaVersionLegacy: + return len(fields) == 3 && len(fields[0]) == base64.StdEncoding.EncodedLen(32) && len(fields[1]) == base64.StdEncoding.EncodedLen(24) && fields[2] != "" + case SchemaVersionHybrid: + return len(fields) == 4 && len(fields[0]) == base64.StdEncoding.EncodedLen(32) && len(fields[1]) == base64.StdEncoding.EncodedLen(hybridMLKEMCiphertextSize) && len(fields[2]) == base64.StdEncoding.EncodedLen(24) && fields[3] != "" + default: + return false + } } -// Dump dumps to the wire format +// Dump dumps to the v1 wire format. func (b *boxedMessage) Dump() []byte { pub := base64.StdEncoding.EncodeToString(b.EncrypterPublic[:]) nonce := base64.StdEncoding.EncodeToString(b.Nonce[:]) @@ -46,55 +61,36 @@ func (b *boxedMessage) Dump() []byte { return []byte(str) } -// Load restores from the wire format. +// Load restores from the v1 wire format. func (b *boxedMessage) Load(from []byte) error { - var ssver, spub, snonce, sbox string - var err error - - allMatches := messageParser.FindAllStringSubmatch(string(from), -1) // -> [][][]byte - if len(allMatches) != 1 { - return fmt.Errorf("invalid message format") - } - matches := allMatches[0] - if len(matches) != 5 { - return fmt.Errorf("invalid message format") - } - - ssver = matches[1] - spub = matches[2] - snonce = matches[3] - sbox = matches[4] - - b.SchemaVersion, err = strconv.Atoi(ssver) + version, fields, err := parseBoxedEnvelope(from) if err != nil { return err } + if version != SchemaVersionLegacy || len(fields) != 3 { + return fmt.Errorf("invalid message format") + } + b.SchemaVersion = version - pub, err := base64.StdEncoding.DecodeString(spub) + pub, err := base64.StdEncoding.DecodeString(fields[0]) if err != nil { return err } - pubBytes := []byte(pub) - if len(pubBytes) != 32 { + if len(pub) != 32 { return fmt.Errorf("public key invalid") } - var public [32]byte - copy(public[:], pubBytes[0:32]) - b.EncrypterPublic = public + copy(b.EncrypterPublic[:], pub) - nnc, err := base64.StdEncoding.DecodeString(snonce) + nnc, err := base64.StdEncoding.DecodeString(fields[1]) if err != nil { return err } - nonceBytes := []byte(nnc) - if len(nonceBytes) != 24 { + if len(nnc) != 24 { return fmt.Errorf("nonce invalid") } - var nonce [24]byte - copy(nonce[:], nonceBytes[0:24]) - b.Nonce = nonce + copy(b.Nonce[:], nnc) - box, err := base64.StdEncoding.DecodeString(sbox) + box, err := base64.StdEncoding.DecodeString(fields[2]) if err != nil { return err } @@ -102,3 +98,23 @@ func (b *boxedMessage) Load(from []byte) error { return nil } + +func parseBoxedEnvelope(data []byte) (version int, fields []string, err error) { + message := string(data) + if !strings.HasPrefix(message, boxedMessagePrefix) || !strings.HasSuffix(message, boxedMessageSuffix) { + return 0, nil, fmt.Errorf("invalid message format") + } + + body := strings.TrimSuffix(strings.TrimPrefix(message, boxedMessagePrefix), boxedMessageSuffix) + versionString, rest, ok := strings.Cut(body, ":") + if !ok || versionString == "" || rest == "" { + return 0, nil, fmt.Errorf("invalid message format") + } + + version, err = strconv.Atoi(versionString) + if err != nil { + return 0, nil, err + } + + return version, strings.Split(rest, ":"), nil +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 63de1c6..0e64e0f 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -1,13 +1,11 @@ -// Package crypto implements a simple convenience wrapper around -// golang.org/x/crypto/nacl/box. It ultimately models a situation where you -// don't care about authenticating the encryptor, so the nonce and encryption -// public key are prepended to the encrypted message. +// Package crypto implements ejson's encrypted value formats. Legacy v1 values +// use golang.org/x/crypto/nacl/box. Hybrid v3 values use X25519 + ML-KEM-768 to +// derive a shared key and XChaCha20-Poly1305 to encrypt the value. // -// Shared key precomputation is used when encrypting but not when decrypting. -// This is not an inherent limitation, but it would complicate the -// implementation a little bit to do precomputation during decryption also. -// If performance becomes an issue (highly unlikely), it's completely feasible -// to add. +// The legacy v1 format models a situation where you don't care about +// authenticating the encryptor, so the nonce and encryption public key are +// prepended to the encrypted message. Shared key precomputation is used when +// encrypting v1 values but not when decrypting. package crypto import ( diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 3261ced..a27abbe 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -63,24 +63,38 @@ func TestRoundtrip(t *testing.T) { }) } -func ExampleEncrypt(peerPublic [32]byte) { - var kp Keypair - if err := kp.Generate(); err != nil { +func ExampleEncrypter_Encrypt() { + var ephemeral, recipient Keypair + if err := ephemeral.Generate(); err != nil { + panic(err) + } + if err := recipient.Generate(); err != nil { panic(err) } - encrypter := kp.Encrypter(peerPublic) + encrypter := ephemeral.Encrypter(recipient.Public) boxed, err := encrypter.Encrypt([]byte("this is my message")) - fmt.Println(boxed, err) + fmt.Println(err == nil, len(boxed) > 0) + // Output: true true } -func ExampleDecrypt(myPublic, myPrivate [32]byte, encrypted []byte) { - kp := Keypair{ - Public: myPublic, - Private: myPrivate, +func ExampleDecrypter_Decrypt() { + var ephemeral, recipient Keypair + if err := ephemeral.Generate(); err != nil { + panic(err) + } + if err := recipient.Generate(); err != nil { + panic(err) + } + + encrypter := ephemeral.Encrypter(recipient.Public) + encrypted, err := encrypter.Encrypt([]byte("this is my message")) + if err != nil { + panic(err) } - decrypter := kp.Decrypter() + decrypter := recipient.Decrypter() plaintext, err := decrypter.Decrypt(encrypted) - fmt.Println(plaintext, err) + fmt.Println(string(plaintext), err) + // Output: this is my message } diff --git a/crypto/hybrid.go b/crypto/hybrid.go new file mode 100644 index 0000000..1f73c09 --- /dev/null +++ b/crypto/hybrid.go @@ -0,0 +1,241 @@ +package crypto + +import ( + "crypto/ecdh" + "crypto/hkdf" + "crypto/mlkem" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "fmt" + + "golang.org/x/crypto/chacha20poly1305" +) + +const ( + hybridKDFSalt = "ejson/v3/x25519-mlkem768/hkdf-sha256" + hybridAADDomain = "ejson/v3/x25519-mlkem768/xchacha20poly1305" +) + +// HybridEncrypter encrypts individual JSON string values using the v3 hybrid +// X25519 + ML-KEM-768 KEM construction and XChaCha20-Poly1305. +type HybridEncrypter struct { + PeerPublic HybridPublicKey + recipientX25519Public *ecdh.PublicKey + mlkemPublic *mlkem.EncapsulationKey768 +} + +// HybridDecrypter decrypts v3 hybrid boxed messages. +type HybridDecrypter struct { + PrivateKey HybridPrivateKey + x25519Private *ecdh.PrivateKey + mlkemPrivate *mlkem.DecapsulationKey768 +} + +type hybridBoxedMessage struct { + SchemaVersion int + EphemeralX25519Public [hybridX25519KeySize]byte + MLKEMCiphertext [hybridMLKEMCiphertextSize]byte + Nonce [chacha20poly1305.NonceSizeX]byte + Box []byte +} + +// NewHybridEncrypter validates and returns a v3 hybrid encrypter. +func NewHybridEncrypter(peerPublic HybridPublicKey) (*HybridEncrypter, error) { + recipientX25519Public, err := ecdh.X25519().NewPublicKey(peerPublic.X25519Public[:]) + if err != nil { + return nil, fmt.Errorf("public key invalid") + } + mlkemPublic, err := mlkem.NewEncapsulationKey768(peerPublic.MLKEMEncapsulationKey[:]) + if err != nil { + return nil, fmt.Errorf("public key invalid") + } + return &HybridEncrypter{PeerPublic: peerPublic, recipientX25519Public: recipientX25519Public, mlkemPublic: mlkemPublic}, nil +} + +// NewHybridDecrypter validates and returns a v3 hybrid decrypter. +func NewHybridDecrypter(privateKey HybridPrivateKey) (*HybridDecrypter, error) { + x25519Private, err := ecdh.X25519().NewPrivateKey(privateKey.X25519Private[:]) + if err != nil { + return nil, fmt.Errorf("invalid private key") + } + mlkemPrivate, err := mlkem.NewDecapsulationKey768(privateKey.MLKEMSeed[:]) + if err != nil { + return nil, fmt.Errorf("invalid private key") + } + return &HybridDecrypter{PrivateKey: privateKey, x25519Private: x25519Private, mlkemPrivate: mlkemPrivate}, nil +} + +// Encrypt takes a plaintext message and returns a v3 hybrid boxed message. If +// the input already looks like any supported ejson boxed message, it is returned +// unchanged to preserve ejson's no-reencrypt behavior. +func (e *HybridEncrypter) Encrypt(message []byte) ([]byte, error) { + if IsBoxedMessage(message) { + return message, nil + } + boxed, err := e.encrypt(message) + if err != nil { + return nil, err + } + return boxed.Dump(), nil +} + +func (e *HybridEncrypter) encrypt(message []byte) (*hybridBoxedMessage, error) { + ephemeralX25519Private, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + x25519SharedSecret, err := ephemeralX25519Private.ECDH(e.recipientX25519Public) + if err != nil { + return nil, err + } + + mlkemSharedSecret, mlkemCiphertext := e.mlkemPublic.Encapsulate() + + ephemeralX25519Public := ephemeralX25519Private.PublicKey().Bytes() + key, aad, err := deriveHybridAEADKeyAndAAD(e.PeerPublic, ephemeralX25519Public, mlkemCiphertext, x25519SharedSecret, mlkemSharedSecret) + if err != nil { + return nil, err + } + + aead, err := chacha20poly1305.NewX(key) + if err != nil { + return nil, err + } + + nonce, err := genNonce() + if err != nil { + return nil, err + } + box := aead.Seal(nil, nonce[:], message, aad) + + var out hybridBoxedMessage + out.SchemaVersion = SchemaVersionHybrid + copy(out.EphemeralX25519Public[:], ephemeralX25519Public) + copy(out.MLKEMCiphertext[:], mlkemCiphertext) + copy(out.Nonce[:], nonce[:]) + out.Box = box + return &out, nil +} + +// Decrypt takes a v3 hybrid boxed message and returns the decrypted plaintext. +func (d *HybridDecrypter) Decrypt(message []byte) ([]byte, error) { + var bm hybridBoxedMessage + if err := bm.Load(message); err != nil { + return nil, err + } + return d.decrypt(&bm) +} + +func (d *HybridDecrypter) decrypt(bm *hybridBoxedMessage) ([]byte, error) { + ephemeralX25519Public, err := ecdh.X25519().NewPublicKey(bm.EphemeralX25519Public[:]) + if err != nil { + return nil, ErrDecryptionFailed + } + x25519SharedSecret, err := d.x25519Private.ECDH(ephemeralX25519Public) + if err != nil { + return nil, ErrDecryptionFailed + } + + mlkemSharedSecret, err := d.mlkemPrivate.Decapsulate(bm.MLKEMCiphertext[:]) + if err != nil { + return nil, ErrDecryptionFailed + } + + key, aad, err := deriveHybridAEADKeyAndAAD(d.PrivateKey.Public, bm.EphemeralX25519Public[:], bm.MLKEMCiphertext[:], x25519SharedSecret, mlkemSharedSecret) + if err != nil { + return nil, ErrDecryptionFailed + } + aead, err := chacha20poly1305.NewX(key) + if err != nil { + return nil, ErrDecryptionFailed + } + + plaintext, err := aead.Open(nil, bm.Nonce[:], bm.Box, aad) + if err != nil { + return nil, ErrDecryptionFailed + } + return plaintext, nil +} + +func (b *hybridBoxedMessage) Dump() []byte { + epk := base64.StdEncoding.EncodeToString(b.EphemeralX25519Public[:]) + kemCiphertext := base64.StdEncoding.EncodeToString(b.MLKEMCiphertext[:]) + nonce := base64.StdEncoding.EncodeToString(b.Nonce[:]) + box := base64.StdEncoding.EncodeToString(b.Box) + + return []byte(fmt.Sprintf("EJ[%d:%s:%s:%s:%s]", b.SchemaVersion, epk, kemCiphertext, nonce, box)) +} + +func (b *hybridBoxedMessage) Load(from []byte) error { + version, fields, err := parseBoxedEnvelope(from) + if err != nil { + return err + } + if version != SchemaVersionHybrid || len(fields) != 4 { + return fmt.Errorf("invalid message format") + } + b.SchemaVersion = version + + ephemeralPublic, err := base64.StdEncoding.DecodeString(fields[0]) + if err != nil { + return err + } + if len(ephemeralPublic) != hybridX25519KeySize { + return fmt.Errorf("public key invalid") + } + copy(b.EphemeralX25519Public[:], ephemeralPublic) + + mlkemCiphertext, err := base64.StdEncoding.DecodeString(fields[1]) + if err != nil { + return err + } + if len(mlkemCiphertext) != hybridMLKEMCiphertextSize { + return fmt.Errorf("ML-KEM ciphertext invalid") + } + copy(b.MLKEMCiphertext[:], mlkemCiphertext) + + nonce, err := base64.StdEncoding.DecodeString(fields[2]) + if err != nil { + return err + } + if len(nonce) != chacha20poly1305.NonceSizeX { + return fmt.Errorf("nonce invalid") + } + copy(b.Nonce[:], nonce) + + box, err := base64.StdEncoding.DecodeString(fields[3]) + if err != nil { + return err + } + b.Box = []byte(box) + return nil +} + +func deriveHybridAEADKeyAndAAD(recipientPublic HybridPublicKey, ephemeralX25519Public, mlkemCiphertext, x25519SharedSecret, mlkemSharedSecret []byte) (key []byte, aad []byte, err error) { + if len(ephemeralX25519Public) != hybridX25519KeySize || len(mlkemCiphertext) != hybridMLKEMCiphertextSize || len(x25519SharedSecret) != hybridX25519KeySize || len(mlkemSharedSecret) != mlkem.SharedKeySize { + return nil, nil, fmt.Errorf("invalid hybrid key material") + } + + ikm := make([]byte, 0, len(x25519SharedSecret)+len(mlkemSharedSecret)) + ikm = append(ikm, x25519SharedSecret...) + ikm = append(ikm, mlkemSharedSecret...) + + aad = hybridTranscript(recipientPublic, ephemeralX25519Public, mlkemCiphertext) + key, err = hkdf.Key(sha256.New, ikm, []byte(hybridKDFSalt), string(aad), chacha20poly1305.KeySize) + if err != nil { + return nil, nil, err + } + return key, aad, nil +} + +func hybridTranscript(recipientPublic HybridPublicKey, ephemeralX25519Public, mlkemCiphertext []byte) []byte { + out := make([]byte, 0, len(hybridAADDomain)+1+len(ephemeralX25519Public)+len(recipientPublic.X25519Public)+len(recipientPublic.MLKEMEncapsulationKey)+len(mlkemCiphertext)) + out = append(out, hybridAADDomain...) + out = append(out, 0) + out = append(out, ephemeralX25519Public...) + out = append(out, recipientPublic.X25519Public[:]...) + out = append(out, recipientPublic.MLKEMEncapsulationKey[:]...) + out = append(out, mlkemCiphertext...) + return out +} diff --git a/crypto/hybrid_test.go b/crypto/hybrid_test.go new file mode 100644 index 0000000..94246f7 --- /dev/null +++ b/crypto/hybrid_test.go @@ -0,0 +1,216 @@ +package crypto + +import ( + "bytes" + "strings" + "testing" +) + +func TestHybridKeypairGenerationAndParsing(t *testing.T) { + pub, priv, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(pub.String(), "v3:") { + t.Fatalf("hybrid public key should have v3 prefix: %q", pub.String()[:3]) + } + if len(pub.KeyID()) != 32 { + t.Fatalf("hybrid key ID should be 32 hex chars, got %d", len(pub.KeyID())) + } + + parsedPub, err := ParsePublicKeyString(pub.String()) + if err != nil { + t.Fatal(err) + } + if parsedPub.Version() != SchemaVersionHybrid { + t.Fatalf("parsed public key version = %d, want %d", parsedPub.Version(), SchemaVersionHybrid) + } + if parsedPub.String() != pub.String() { + t.Fatal("parsed public key did not roundtrip") + } + + parsedPriv, err := ParsePrivateKeyForPublic(parsedPub, []byte(priv.String())) + if err != nil { + t.Fatal(err) + } + if parsedPriv.Version() != SchemaVersionHybrid { + t.Fatalf("parsed private key version = %d, want %d", parsedPriv.Version(), SchemaVersionHybrid) + } + + enc, err := NewMessageEncrypter(parsedPub) + if err != nil { + t.Fatal(err) + } + dec, err := NewMessageDecrypter(parsedPub, parsedPriv) + if err != nil { + t.Fatal(err) + } + ct, err := enc.Encrypt([]byte("secret")) + if err != nil { + t.Fatal(err) + } + pt, err := dec.Decrypt(ct) + if err != nil { + t.Fatal(err) + } + if string(pt) != "secret" { + t.Fatalf("plaintext = %q, want secret", pt) + } +} + +func TestHybridRoundtripAndNoReencrypt(t *testing.T) { + pub, priv, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + enc, err := NewMessageEncrypter(pub) + if err != nil { + t.Fatal(err) + } + dec, err := NewMessageDecrypter(pub, priv) + if err != nil { + t.Fatal(err) + } + + messages := [][]byte{ + []byte(""), + []byte("This is a test of the post-quantum emergency broadcast system."), + bytes.Repeat([]byte("0123456789abcdef"), 640), + } + for _, message := range messages { + ct, err := enc.Encrypt(message) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(ct, []byte("EJ[3:")) { + t.Fatalf("ciphertext prefix = %.5q, want EJ[3:", ct) + } + if !IsBoxedMessage(ct) { + t.Fatalf("v3 ciphertext was not recognized as boxed: %.32q", ct) + } + ct2, err := enc.Encrypt(ct) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(ct2, ct) { + t.Fatal("already boxed v3 message was re-encrypted") + } + + pt, err := dec.Decrypt(ct) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(pt, message) { + t.Fatal("plaintext did not roundtrip") + } + } +} + +func TestHybridRejectsWrongKeyAndTampering(t *testing.T) { + pub, priv, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + wrongPub, wrongPriv, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + enc, err := NewMessageEncrypter(pub) + if err != nil { + t.Fatal(err) + } + dec, err := NewMessageDecrypter(pub, priv) + if err != nil { + t.Fatal(err) + } + wrongDec, err := NewMessageDecrypter(wrongPub, wrongPriv) + if err != nil { + t.Fatal(err) + } + + ct, err := enc.Encrypt([]byte("swordfish")) + if err != nil { + t.Fatal(err) + } + if _, err := wrongDec.Decrypt(ct); err == nil { + t.Fatal("decrypt with wrong key succeeded") + } + + mutations := map[string]func(*hybridBoxedMessage){ + "ephemeral X25519 public key": func(bm *hybridBoxedMessage) { bm.EphemeralX25519Public[0] ^= 0x80 }, + "ML-KEM ciphertext": func(bm *hybridBoxedMessage) { bm.MLKEMCiphertext[0] ^= 0x80 }, + "nonce": func(bm *hybridBoxedMessage) { bm.Nonce[0] ^= 0x80 }, + "AEAD ciphertext": func(bm *hybridBoxedMessage) { bm.Box[0] ^= 0x80 }, + } + for name, mutate := range mutations { + var bm hybridBoxedMessage + if err := bm.Load(ct); err != nil { + t.Fatal(err) + } + mutate(&bm) + if _, err := dec.Decrypt(bm.Dump()); err == nil { + t.Fatalf("decrypt succeeded after tampering with %s", name) + } + } +} + +func TestHybridRejectsKEMCiphertextSplicing(t *testing.T) { + pub, priv, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + enc, err := NewMessageEncrypter(pub) + if err != nil { + t.Fatal(err) + } + dec, err := NewMessageDecrypter(pub, priv) + if err != nil { + t.Fatal(err) + } + + ct1, err := enc.Encrypt([]byte("first secret")) + if err != nil { + t.Fatal(err) + } + ct2, err := enc.Encrypt([]byte("second secret")) + if err != nil { + t.Fatal(err) + } + + var bm1, bm2 hybridBoxedMessage + if err := bm1.Load(ct1); err != nil { + t.Fatal(err) + } + if err := bm2.Load(ct2); err != nil { + t.Fatal(err) + } + bm1.MLKEMCiphertext = bm2.MLKEMCiphertext + + if _, err := dec.Decrypt(bm1.Dump()); err == nil { + t.Fatal("decrypt succeeded after ML-KEM ciphertext splice") + } +} + +func TestHybridPrivateKeyRejectsMismatchedPublicKey(t *testing.T) { + pub, _, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + _, otherPriv, err := GenerateHybridKeypair() + if err != nil { + t.Fatal(err) + } + + if _, err := ParsePrivateKeyForPublic(pub, []byte(otherPriv.String())); err == nil { + t.Fatal("mismatched v3 private key parsed successfully") + } +} + +func TestHybridPublicKeyRejectsInvalidInputs(t *testing.T) { + if _, err := ParsePublicKeyString("v3:not base64"); err == nil { + t.Fatal("invalid base64 v3 public key parsed successfully") + } + if _, err := ParsePublicKeyString("v3:" + strings.Repeat("A", 16)); err == nil { + t.Fatal("short v3 public key parsed successfully") + } +} diff --git a/crypto/keys.go b/crypto/keys.go new file mode 100644 index 0000000..4ce7be1 --- /dev/null +++ b/crypto/keys.go @@ -0,0 +1,389 @@ +package crypto + +import ( + "bytes" + "crypto/ecdh" + "crypto/mlkem" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "strings" +) + +const ( + SchemaVersionLegacy = 1 + SchemaVersionHybrid = 3 + + SchemeLegacy = "v1" + SchemeHybrid = "v3" + + hybridPublicKeyPrefix = "v3:" + hybridKeyFileHeader = "ejson-key v3" + hybridKeyIDDomain = "ejson/v3/pubkey" +) + +const ( + hybridX25519KeySize = 32 + hybridMLKEMSeedSize = mlkem.SeedSize + hybridMLKEMPublicKeySize = mlkem.EncapsulationKeySize768 + hybridMLKEMCiphertextSize = mlkem.CiphertextSize768 + hybridPublicKeyPayloadSize = hybridX25519KeySize + hybridMLKEMPublicKeySize +) + +// PublicKey is a schema-aware ejson public key. +type PublicKey interface { + Version() int + String() string + KeyID() string +} + +// PrivateKey is a schema-aware ejson private key. +type PrivateKey interface { + Version() int + String() string +} + +// MessageEncrypter encrypts individual JSON string values. +type MessageEncrypter interface { + Encrypt([]byte) ([]byte, error) +} + +// MessageDecrypter decrypts individual JSON string values. +type MessageDecrypter interface { + Decrypt([]byte) ([]byte, error) +} + +// LegacyPublicKey is the v1 32-byte Curve25519 public key. +type LegacyPublicKey struct { + Key [32]byte +} + +func (k LegacyPublicKey) Version() int { return SchemaVersionLegacy } +func (k LegacyPublicKey) String() string { + return hex.EncodeToString(k.Key[:]) +} +func (k LegacyPublicKey) KeyID() string { return k.String() } + +// LegacyPrivateKey is the v1 32-byte Curve25519 private key. +type LegacyPrivateKey struct { + Key [32]byte +} + +func (k LegacyPrivateKey) Version() int { return SchemaVersionLegacy } +func (k LegacyPrivateKey) String() string { + return hex.EncodeToString(k.Key[:]) +} + +// HybridPublicKey is a v3 X25519 + ML-KEM-768 public key. +type HybridPublicKey struct { + X25519Public [hybridX25519KeySize]byte + MLKEMEncapsulationKey [hybridMLKEMPublicKeySize]byte +} + +func (k HybridPublicKey) Version() int { return SchemaVersionHybrid } +func (k HybridPublicKey) String() string { + return hybridPublicKeyPrefix + base64.StdEncoding.EncodeToString(k.Bytes()) +} +func (k HybridPublicKey) KeyID() string { + h := sha256.New() + _, _ = h.Write([]byte(hybridKeyIDDomain)) + _, _ = h.Write(k.X25519Public[:]) + _, _ = h.Write(k.MLKEMEncapsulationKey[:]) + return hex.EncodeToString(h.Sum(nil)[:16]) +} +func (k HybridPublicKey) Bytes() []byte { + out := make([]byte, 0, hybridPublicKeyPayloadSize) + out = append(out, k.X25519Public[:]...) + out = append(out, k.MLKEMEncapsulationKey[:]...) + return out +} + +// HybridPrivateKey is a v3 X25519 + ML-KEM-768 private key. +type HybridPrivateKey struct { + Public HybridPublicKey + X25519Private [hybridX25519KeySize]byte + MLKEMSeed [hybridMLKEMSeedSize]byte +} + +func (k HybridPrivateKey) Version() int { return SchemaVersionHybrid } +func (k HybridPrivateKey) String() string { + return strings.Join([]string{ + hybridKeyFileHeader, + "keyid: " + k.Public.KeyID(), + "pub-x25519: " + hex.EncodeToString(k.Public.X25519Public[:]), + "pub-mlkem768: " + base64.StdEncoding.EncodeToString(k.Public.MLKEMEncapsulationKey[:]), + "priv-x25519: " + hex.EncodeToString(k.X25519Private[:]), + "priv-mlkem768-seed: " + hex.EncodeToString(k.MLKEMSeed[:]), + }, "\n") +} + +// ParsePublicKeyString parses a public key in ejson document form. Legacy v1 +// keys are bare 64-character hex strings. Hybrid v3 keys are v3:. +func ParsePublicKeyString(s string) (PublicKey, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, hybridPublicKeyPrefix) { + return ParseHybridPublicKeyString(s) + } + return ParseLegacyPublicKeyString(s) +} + +// ParseLegacyPublicKeyString parses a legacy v1 public key. +func ParseLegacyPublicKeyString(s string) (LegacyPublicKey, error) { + var key LegacyPublicKey + if len(s) != 64 { + return key, fmt.Errorf("public key invalid") + } + bs, err := hex.DecodeString(s) + if err != nil { + return key, fmt.Errorf("public key invalid") + } + if len(bs) != len(key.Key) { + return key, fmt.Errorf("public key invalid") + } + copy(key.Key[:], bs) + return key, nil +} + +// ParseHybridPublicKeyString parses a v3 hybrid public key. +func ParseHybridPublicKeyString(s string) (HybridPublicKey, error) { + var key HybridPublicKey + if !strings.HasPrefix(s, hybridPublicKeyPrefix) { + return key, fmt.Errorf("public key invalid") + } + payload, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(s, hybridPublicKeyPrefix)) + if err != nil { + return key, fmt.Errorf("public key invalid") + } + return NewHybridPublicKey(payload) +} + +// NewHybridPublicKey validates and constructs a v3 hybrid public key from raw +// pk_x25519 || ek_mlkem768 bytes. +func NewHybridPublicKey(payload []byte) (HybridPublicKey, error) { + var key HybridPublicKey + if len(payload) != hybridPublicKeyPayloadSize { + return key, fmt.Errorf("public key invalid") + } + + x25519Public := payload[:hybridX25519KeySize] + mlkemPublic := payload[hybridX25519KeySize:] + if _, err := ecdh.X25519().NewPublicKey(x25519Public); err != nil { + return key, fmt.Errorf("public key invalid") + } + if _, err := mlkem.NewEncapsulationKey768(mlkemPublic); err != nil { + return key, fmt.Errorf("public key invalid") + } + + copy(key.X25519Public[:], x25519Public) + copy(key.MLKEMEncapsulationKey[:], mlkemPublic) + return key, nil +} + +// GenerateKeypairForScheme generates a v1 or v3 ejson keypair. +func GenerateKeypairForScheme(scheme string) (PublicKey, PrivateKey, error) { + switch normalizeScheme(scheme) { + case SchemeLegacy: + var kp Keypair + if err := kp.Generate(); err != nil { + return nil, nil, err + } + return LegacyPublicKey{Key: kp.Public}, LegacyPrivateKey{Key: kp.Private}, nil + case SchemeHybrid: + return GenerateHybridKeypair() + default: + return nil, nil, fmt.Errorf("unsupported key scheme %q", scheme) + } +} + +// GenerateHybridKeypair generates a v3 X25519 + ML-KEM-768 keypair. +func GenerateHybridKeypair() (HybridPublicKey, HybridPrivateKey, error) { + var pub HybridPublicKey + var priv HybridPrivateKey + + x25519Private, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return pub, priv, err + } + x25519Public := x25519Private.PublicKey().Bytes() + + mlkemPrivate, err := mlkem.GenerateKey768() + if err != nil { + return pub, priv, err + } + mlkemSeed := mlkemPrivate.Bytes() + mlkemPublic := mlkemPrivate.EncapsulationKey().Bytes() + + copy(pub.X25519Public[:], x25519Public) + copy(pub.MLKEMEncapsulationKey[:], mlkemPublic) + priv.Public = pub + copy(priv.X25519Private[:], x25519Private.Bytes()) + copy(priv.MLKEMSeed[:], mlkemSeed) + return pub, priv, nil +} + +// ParsePrivateKeyForPublic parses private key material for the corresponding +// public key and verifies that v3 keyfiles match the public key. +func ParsePrivateKeyForPublic(pub PublicKey, data []byte) (PrivateKey, error) { + switch p := pub.(type) { + case LegacyPublicKey: + return parseLegacyPrivateKey(data) + case HybridPublicKey: + return parseHybridPrivateKeyForPublic(p, data) + default: + return nil, fmt.Errorf("unsupported key scheme") + } +} + +// NewMessageEncrypter returns an encrypter for the given schema-aware public key. +func NewMessageEncrypter(pub PublicKey) (MessageEncrypter, error) { + switch p := pub.(type) { + case LegacyPublicKey: + var kp Keypair + if err := kp.Generate(); err != nil { + return nil, err + } + return kp.Encrypter(p.Key), nil + case HybridPublicKey: + return NewHybridEncrypter(p) + default: + return nil, fmt.Errorf("unsupported key scheme") + } +} + +// NewMessageDecrypter returns a decrypter for the given schema-aware keypair. +func NewMessageDecrypter(pub PublicKey, priv PrivateKey) (MessageDecrypter, error) { + if pub.Version() != priv.Version() { + return nil, fmt.Errorf("private key does not match public key") + } + + switch p := pub.(type) { + case LegacyPublicKey: + legacyPriv, ok := priv.(LegacyPrivateKey) + if !ok { + return nil, fmt.Errorf("private key does not match public key") + } + kp := Keypair{Public: p.Key, Private: legacyPriv.Key} + return kp.Decrypter(), nil + case HybridPublicKey: + hybridPriv, ok := priv.(HybridPrivateKey) + if !ok { + return nil, fmt.Errorf("private key does not match public key") + } + if hybridPriv.Public.String() != p.String() { + return nil, fmt.Errorf("private key does not match public key") + } + return NewHybridDecrypter(hybridPriv) + default: + return nil, fmt.Errorf("unsupported key scheme") + } +} + +func parseLegacyPrivateKey(data []byte) (LegacyPrivateKey, error) { + var key LegacyPrivateKey + bs, err := hex.DecodeString(strings.TrimSpace(string(data))) + if err != nil { + return key, err + } + if len(bs) != len(key.Key) { + return key, fmt.Errorf("invalid private key") + } + copy(key.Key[:], bs) + return key, nil +} + +func parseHybridPrivateKeyForPublic(expected HybridPublicKey, data []byte) (HybridPrivateKey, error) { + fields, err := parseHybridPrivateKeyFields(data) + if err != nil { + return HybridPrivateKey{}, err + } + + keyID := fields["keyid"] + pubXString := fields["pub-x25519"] + pubMLKEMString := fields["pub-mlkem768"] + privXString := fields["priv-x25519"] + seedString := fields["priv-mlkem768-seed"] + if keyID == "" || pubXString == "" || pubMLKEMString == "" || privXString == "" || seedString == "" { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + + pubX, err := hex.DecodeString(pubXString) + if err != nil || len(pubX) != hybridX25519KeySize { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + pubMLKEM, err := base64.StdEncoding.DecodeString(pubMLKEMString) + if err != nil || len(pubMLKEM) != hybridMLKEMPublicKeySize { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + payload := append(append([]byte{}, pubX...), pubMLKEM...) + public, err := NewHybridPublicKey(payload) + if err != nil { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + if keyID != public.KeyID() || public.String() != expected.String() { + return HybridPrivateKey{}, fmt.Errorf("private key does not match public key") + } + + privX, err := hex.DecodeString(privXString) + if err != nil || len(privX) != hybridX25519KeySize { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + x25519Private, err := ecdh.X25519().NewPrivateKey(privX) + if err != nil { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + if !bytes.Equal(x25519Private.PublicKey().Bytes(), public.X25519Public[:]) { + return HybridPrivateKey{}, fmt.Errorf("private key does not match public key") + } + + seed, err := hex.DecodeString(seedString) + if err != nil || len(seed) != hybridMLKEMSeedSize { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + mlkemPrivate, err := mlkem.NewDecapsulationKey768(seed) + if err != nil { + return HybridPrivateKey{}, fmt.Errorf("invalid private key") + } + if !bytes.Equal(mlkemPrivate.EncapsulationKey().Bytes(), public.MLKEMEncapsulationKey[:]) { + return HybridPrivateKey{}, fmt.Errorf("private key does not match public key") + } + + var private HybridPrivateKey + private.Public = public + copy(private.X25519Private[:], privX) + copy(private.MLKEMSeed[:], seed) + return private, nil +} + +func parseHybridPrivateKeyFields(data []byte) (map[string]string, error) { + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) == 0 || strings.TrimSpace(lines[0]) != hybridKeyFileHeader { + return nil, fmt.Errorf("invalid private key") + } + + fields := make(map[string]string, len(lines)-1) + for _, line := range lines[1:] { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, value, ok := strings.Cut(line, ":") + if !ok { + return nil, fmt.Errorf("invalid private key") + } + fields[strings.TrimSpace(name)] = strings.TrimSpace(value) + } + return fields, nil +} + +func normalizeScheme(scheme string) string { + switch strings.ToLower(strings.TrimSpace(scheme)) { + case "", "1", SchemeLegacy, "legacy", "classic", "nacl", "box": + return SchemeLegacy + case "3", SchemeHybrid, "hybrid", "pqc", "post-quantum", "postquantum", "mlkem", "ml-kem": + return SchemeHybrid + default: + return strings.ToLower(strings.TrimSpace(scheme)) + } +} diff --git a/dev.yml b/dev.yml index c9bb7d6..4457a06 100644 --- a/dev.yml +++ b/dev.yml @@ -6,7 +6,7 @@ up: - goreleaser - ruby - go: - version: '1.22.3' + version: '1.26.3' - bundler commands: diff --git a/ejson.go b/ejson.go index db8f034..3fd2252 100644 --- a/ejson.go +++ b/ejson.go @@ -5,19 +5,17 @@ package ejson import ( "bytes" - "encoding/hex" "fmt" "io" "os" - "strings" "github.com/Shopify/ejson/crypto" "github.com/Shopify/ejson/json" ) -// GenerateKeypair is used to create a new ejson keypair. It returns the keys as -// hex-encoded strings, suitable for printing to the screen. hex.DecodeString -// can be used to load the true representation if necessary. +// GenerateKeypair is used to create a new legacy v1 ejson keypair. It returns +// the keys as hex-encoded strings, suitable for printing to the screen. +// hex.DecodeString can be used to load the true representation if necessary. func GenerateKeypair() (pub string, priv string, err error) { var kp crypto.Keypair if err := kp.Generate(); err != nil { @@ -26,6 +24,17 @@ func GenerateKeypair() (pub string, priv string, err error) { return kp.PublicString(), kp.PrivateString(), nil } +// GenerateKeypairForScheme is used to create a new ejson keypair for the named +// scheme. The returned public key is suitable for the _public_key field. The +// returned private key is suitable for writing into the keydir under keyID. +func GenerateKeypairForScheme(scheme string) (pub string, priv string, keyID string, err error) { + publicKey, privateKey, err := crypto.GenerateKeypairForScheme(scheme) + if err != nil { + return "", "", "", err + } + return publicKey.String(), privateKey.String(), publicKey.KeyID(), nil +} + // Encrypt reads all contents from 'in', extracts the pubkey // and performs the requested encryption operation, writing // the resulting data to 'out'. @@ -37,22 +46,20 @@ func Encrypt(in io.Reader, out io.Writer) (int, error) { return -1, err } - var myKP crypto.Keypair - if err = myKP.Generate(); err != nil { + data, err = json.CollapseMultilineStringLiterals(data) + if err != nil { return -1, err } - data, err = json.CollapseMultilineStringLiterals(data) + pubkey, err := json.ExtractCryptoPublicKey(data) if err != nil { return -1, err } - pubkey, err := json.ExtractPublicKey(data) + encrypter, err := crypto.NewMessageEncrypter(pubkey) if err != nil { return -1, err } - - encrypter := myKP.Encrypter(pubkey) walker := json.Walker{ Action: encrypter.Encrypt, } @@ -68,7 +75,7 @@ func Encrypt(in io.Reader, out io.Writer) (int, error) { // EncryptFileInPlace takes a path to a file on disk, which must be a valid EJSON file // (see README.md for more on what constitutes a valid EJSON file). Any // encryptable-but-unencrypted fields in the file will be encrypted using the -// public key embdded in the file, and the resulting text will be written over +// public key embedded in the file, and the resulting text will be written over // the file present on disk. func EncryptFileInPlace(filePath string) (int, error) { var fileMode os.FileMode @@ -110,7 +117,7 @@ func Decrypt(in io.Reader, out io.Writer, keydir string, userSuppliedPrivateKey return err } - pubkey, err := json.ExtractPublicKey(data) + pubkey, err := json.ExtractCryptoPublicKey(data) if err != nil { return err } @@ -120,12 +127,10 @@ func Decrypt(in io.Reader, out io.Writer, keydir string, userSuppliedPrivateKey return err } - myKP := crypto.Keypair{ - Public: pubkey, - Private: privkey, + decrypter, err := crypto.NewMessageDecrypter(pubkey, privkey) + if err != nil { + return err } - - decrypter := myKP.Decrypter() walker := json.Walker{ Action: decrypter.Decrypt, } @@ -143,9 +148,8 @@ func Decrypt(in io.Reader, out io.Writer, keydir string, userSuppliedPrivateKey // DecryptFile takes a path to an encrypted EJSON file and returns the data // decrypted. The public key used to encrypt the values is embedded in the // referenced document, and the matching private key is searched for in keydir. -// There must exist a file in keydir whose name is the public key from the -// EJSON document, and whose contents are the corresponding private key. See -// README.md for more details on this. +// For legacy v1 keys, the keydir filename is the public key. For v3 hybrid +// keys, the keydir filename is the public key's short key ID. func DecryptFile(filePath, keydir string, userSuppliedPrivateKey string) ([]byte, error) { if _, err := os.Stat(filePath); err != nil { return nil, err @@ -164,8 +168,8 @@ func DecryptFile(filePath, keydir string, userSuppliedPrivateKey string) ([]byte return outBuffer.Bytes(), err } -func readPrivateKeyFromDisk(pubkey [32]byte, keydir string) (privkey string, err error) { - keyFile := fmt.Sprintf("%s/%x", keydir, pubkey) +func readPrivateKeyFromDisk(pubkey crypto.PublicKey, keydir string) (privkey string, err error) { + keyFile := fmt.Sprintf("%s/%s", keydir, pubkey.KeyID()) var fileContents []byte fileContents, err = os.ReadFile(keyFile) if err != nil { @@ -176,26 +180,17 @@ func readPrivateKeyFromDisk(pubkey [32]byte, keydir string) (privkey string, err return } -func findPrivateKey(pubkey [32]byte, keydir string, userSuppliedPrivateKey string) (privkey [32]byte, err error) { +func findPrivateKey(pubkey crypto.PublicKey, keydir string, userSuppliedPrivateKey string) (crypto.PrivateKey, error) { var privkeyString string if userSuppliedPrivateKey != "" { privkeyString = userSuppliedPrivateKey } else { + var err error privkeyString, err = readPrivateKeyFromDisk(pubkey, keydir) if err != nil { - return privkey, err + return nil, err } } - privkeyBytes, err := hex.DecodeString(strings.TrimSpace(privkeyString)) - if err != nil { - return - } - - if len(privkeyBytes) != 32 { - err = fmt.Errorf("invalid private key") - return - } - copy(privkey[:], privkeyBytes) - return + return crypto.ParsePrivateKeyForPublic(pubkey, []byte(privkeyString)) } diff --git a/ejson_test.go b/ejson_test.go index 92115ce..704c1f4 100644 --- a/ejson_test.go +++ b/ejson_test.go @@ -4,6 +4,7 @@ import ( "os" "path" "regexp" + "strings" "testing" . "github.com/smartystreets/goconvey/convey" @@ -29,6 +30,26 @@ func TestGenerateKeypair(t *testing.T) { }) } +func TestGenerateKeypairForScheme(t *testing.T) { + Convey("GenerateKeypairForScheme", t, func() { + Convey("generates legacy v1 keys by default", func() { + pub, priv, keyID, err := GenerateKeypairForScheme("v1") + So(err, ShouldBeNil) + So(pub, ShouldHaveLength, 64) + So(priv, ShouldHaveLength, 64) + So(keyID, ShouldEqual, pub) + }) + + Convey("generates hybrid v3 keys", func() { + pub, priv, keyID, err := GenerateKeypairForScheme("v3") + So(err, ShouldBeNil) + So(strings.HasPrefix(pub, "v3:"), ShouldBeTrue) + So(strings.HasPrefix(priv, "ejson-key v3\n"), ShouldBeTrue) + So(keyID, ShouldHaveLength, 32) + }) + }) +} + func setData(path string, data []byte) error { tmpFile, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0o600) if err != nil { @@ -111,6 +132,52 @@ func TestEncryptFileInPlace(t *testing.T) { }) } +func TestHybridEncryptDecryptFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "ejson_hybrid_keys") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pub, priv, keyID, err := GenerateKeypairForScheme("v3") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path.Join(tempDir, keyID), []byte(priv), 0o600); err != nil { + t.Fatal(err) + } + + tempFile, err := os.CreateTemp(tempDir, "ejson_hybrid_test") + if err != nil { + t.Fatal(err) + } + tempFileName := tempFile.Name() + if err := tempFile.Close(); err != nil { + t.Fatal(err) + } + + Convey("hybrid v3 EncryptFileInPlace and DecryptFile", t, func() { + original := `{"_public_key": "` + pub + `", "a": "b", "nested": {"secret": "value"}, "_comment": "plaintext"}` + setData(tempFileName, []byte(original)) + + _, err := EncryptFileInPlace(tempFileName) + So(err, ShouldBeNil) + ciphertext, err := os.ReadFile(tempFileName) + So(err, ShouldBeNil) + So(string(ciphertext), ShouldContainSubstring, `"a": "EJ[3:`) + So(string(ciphertext), ShouldContainSubstring, `"secret": "EJ[3:`) + So(string(ciphertext), ShouldContainSubstring, `"_comment": "plaintext"`) + + out, err := DecryptFile(tempFileName, tempDir, "") + So(err, ShouldBeNil) + So(string(out), ShouldEqual, original) + + out, err = DecryptFile(tempFileName, "/does/not/matter", priv) + So(err, ShouldBeNil) + So(string(out), ShouldEqual, original) + }) +} + func TestDecryptFile(t *testing.T) { tempDir, err := os.MkdirTemp("", "ejson_keys") if err != nil { diff --git a/json/key.go b/json/key.go index 66177c8..997a531 100644 --- a/json/key.go +++ b/json/key.go @@ -2,8 +2,10 @@ package json import ( "encoding/hex" - "encoding/json" + stdjson "encoding/json" "errors" + + ejsoncrypto "github.com/Shopify/ejson/crypto" ) const ( @@ -21,42 +23,53 @@ var ErrPublicKeyMissing = errors.New("public key not present in EJSON file") var ErrPublicKeyInvalid = errors.New("public key has invalid format") // ExtractPublicKey finds the _public_key value in an EJSON document and -// parses it into a key usable with the crypto library. +// parses it into a legacy v1 key usable with the crypto library. func ExtractPublicKey(data []byte) (key [32]byte, err error) { - var ( - obj map[string]interface{} - ks string - ok bool - bs []byte - ) - err = json.Unmarshal(data, &obj) + ks, err := extractPublicKeyString(data) if err != nil { - return - } - k, ok := obj[PublicKeyField] - if !ok { - goto missing - } - ks, ok = k.(string) - if !ok { - goto invalid + return key, err } + if len(ks) != 64 { - goto invalid + return key, ErrPublicKeyInvalid } - bs, err = hex.DecodeString(ks) + bs, err := hex.DecodeString(ks) if err != nil { - goto invalid + return key, ErrPublicKeyInvalid } if len(bs) != 32 { - goto invalid + return key, ErrPublicKeyInvalid } copy(key[:], bs) - return -missing: - err = ErrPublicKeyMissing - return -invalid: - err = ErrPublicKeyInvalid - return + return key, nil +} + +// ExtractCryptoPublicKey finds the _public_key value in an EJSON document and +// parses it into a schema-aware key usable with the crypto library. +func ExtractCryptoPublicKey(data []byte) (ejsoncrypto.PublicKey, error) { + ks, err := extractPublicKeyString(data) + if err != nil { + return nil, err + } + key, err := ejsoncrypto.ParsePublicKeyString(ks) + if err != nil { + return nil, ErrPublicKeyInvalid + } + return key, nil +} + +func extractPublicKeyString(data []byte) (string, error) { + var obj map[string]interface{} + if err := stdjson.Unmarshal(data, &obj); err != nil { + return "", err + } + k, ok := obj[PublicKeyField] + if !ok { + return "", ErrPublicKeyMissing + } + ks, ok := k.(string) + if !ok { + return "", ErrPublicKeyInvalid + } + return ks, nil } diff --git a/json/key_test.go b/json/key_test.go index 7cb7c7f..d553574 100644 --- a/json/key_test.go +++ b/json/key_test.go @@ -3,6 +3,7 @@ package json import ( "testing" + ejsoncrypto "github.com/Shopify/ejson/crypto" . "github.com/smartystreets/goconvey/convey" ) @@ -15,6 +16,16 @@ func TestKeyExtraction(t *testing.T) { So(err, ShouldBeNil) So(key, ShouldResemble, expected) }) + Convey("extracts schema-aware v3 public keys", func() { + pub, _, err := ejsoncrypto.GenerateHybridKeypair() + So(err, ShouldBeNil) + in := `{"_public_key": "` + pub.String() + `"}` + key, err := ExtractCryptoPublicKey([]byte(in)) + So(err, ShouldBeNil) + So(key.Version(), ShouldEqual, ejsoncrypto.SchemaVersionHybrid) + So(key.String(), ShouldEqual, pub.String()) + }) + Convey("fails", func() { Convey("if key is too short", func() { in := `{"_public_key": "6d79b7e50073e5e66a4581ed08bf1d9a03806cc4648cffeb6df71b5775e5eb0"}` @@ -33,6 +44,12 @@ func TestKeyExtraction(t *testing.T) { _, err := ExtractPublicKey([]byte(in)) So(err, ShouldEqual, ErrPublicKeyMissing) }) + + Convey("or if schema-aware v3 key is malformed", func() { + in := `{"_public_key": "v3:not base64"}` + _, err := ExtractCryptoPublicKey([]byte(in)) + So(err, ShouldEqual, ErrPublicKeyInvalid) + }) }) }) }