diff --git a/README.md b/README.md index f0fedb7d9..54426d22e 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ API and command-line option may change frequently.*** - [HunyuanVideo 1.5](./docs/hunyuan_video.md) - [LingBot-Video](./docs/lingbot_video.md) - [PhotoMaker](./docs/photo_maker.md) support. - - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL) + - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus) - Control Net support with SD 1.5 - [ADetailer](./docs/adetailer.md) - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora) diff --git a/docs/ip_adapter.md b/docs/ip_adapter.md index 23a5e3d0e..76a651011 100644 --- a/docs/ip_adapter.md +++ b/docs/ip_adapter.md @@ -11,6 +11,10 @@ through a decoupled cross-attention added to every attn2 layer of the UNet. It composes with Control Net, so a reference image (appearance) and an OpenPose hint (pose) can be combined in a single generation. +Both the classic adapters and the higher-fidelity **Plus** adapters are +supported; see [Plus variants](#plus-variants) below. The variant is +detected from the weight file, so the same options work for both. + ## Required weights 1. A base SD 1.5 or SDXL model. @@ -21,6 +25,11 @@ an OpenPose hint (pose) can be combined in a single generation. [h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter): - SD 1.5: `models/ip-adapter_sd15.safetensors` - SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors` + - SD 1.5 Plus: `models/ip-adapter-plus_sd15.safetensors` + - SDXL Plus: `sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors` + + The Plus files (`ip-adapter-plus_*`) are used exactly like the classic + ones; see [Plus variants](#plus-variants). ## Options @@ -45,6 +54,29 @@ sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safet The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and `--offload-to-cpu`) on GPUs with limited VRAM. +## Plus variants + +The Plus adapters (`ip-adapter-plus_sd15`, `ip-adapter-plus_sdxl_vit-h`) +replace the small linear image projection with a Resampler (a +Perceiver-style module with learned latent queries). Instead of pooling the +CLIP-Vision output into one vector, the Resampler attends over the full grid +of penultimate CLIP-Vision hidden states and emits more image tokens (16 +instead of 4). The result transfers finer detail and layout from the +reference, at a small extra cost in the image-projection step. + +No extra flags are needed. The variant is detected from the weight file (the +Resampler's `image_proj.latents` tensor), and every Resampler dimension is +read from the tensor shapes, so the same `--ip-adapter`, +`--ip-adapter-image`, and `--ip-adapter-strength` options apply. Plus +composes with Control Net in the same way as the classic adapters. + +``` +sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter-plus_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512 +``` + +The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the +classic adapters) confirms a Plus file was loaded. + ## Combining with Control Net Add the usual Control Net options to keep the reference appearance while diff --git a/src/model/adapter/ip_adapter.hpp b/src/model/adapter/ip_adapter.hpp index 35fd1bb88..8a0321f49 100644 --- a/src/model/adapter/ip_adapter.hpp +++ b/src/model/adapter/ip_adapter.hpp @@ -31,8 +31,92 @@ namespace IPAdapter { } }; + struct Resampler : public GGMLBlock { + int64_t dim = 1280; + int64_t depth = 4; + int64_t num_queries = 16; + int64_t embed_dim = 1280; + int64_t output_dim = 2048; + int64_t ff_inner = 5120; + int64_t dim_head = 64; + int64_t heads = 20; + + void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { + params["latents"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, dim, num_queries, 1); + } + + Resampler() {} + Resampler(int64_t dim, int64_t depth, int64_t num_queries, int64_t embed_dim, int64_t output_dim, int64_t ff_inner) + : dim(dim), depth(depth), num_queries(num_queries), embed_dim(embed_dim), output_dim(output_dim), ff_inner(ff_inner) { + heads = dim / dim_head; + blocks["proj_in"] = std::shared_ptr(new Linear(embed_dim, dim, true)); + blocks["proj_out"] = std::shared_ptr(new Linear(dim, output_dim, true)); + blocks["norm_out"] = std::shared_ptr(new LayerNorm(output_dim)); + for (int64_t i = 0; i < depth; i++) { + std::string p = "layers." + std::to_string(i); + blocks[p + ".0.norm1"] = std::shared_ptr(new LayerNorm(dim)); + blocks[p + ".0.norm2"] = std::shared_ptr(new LayerNorm(dim)); + blocks[p + ".0.to_q"] = std::shared_ptr(new Linear(dim, dim, false)); + blocks[p + ".0.to_kv"] = std::shared_ptr(new Linear(dim, dim * 2, false)); + blocks[p + ".0.to_out"] = std::shared_ptr(new Linear(dim, dim, false)); + blocks[p + ".1.0"] = std::shared_ptr(new LayerNorm(dim)); + blocks[p + ".1.1"] = std::shared_ptr(new Linear(dim, ff_inner, false)); + blocks[p + ".1.3"] = std::shared_ptr(new Linear(ff_inner, dim, false)); + } + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) { + int64_t N = image_embeds->ne[2]; + auto proj_in = std::dynamic_pointer_cast(blocks["proj_in"]); + auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + + ggml_tensor* x = proj_in->forward(ctx, image_embeds); + ggml_tensor* latents = params["latents"]; + if (N > 1) { + latents = ggml_repeat(ctx->ggml_ctx, latents, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, dim, num_queries, N)); + } + + for (int64_t i = 0; i < depth; i++) { + std::string p = "layers." + std::to_string(i); + auto norm1 = std::dynamic_pointer_cast(blocks[p + ".0.norm1"]); + auto norm2 = std::dynamic_pointer_cast(blocks[p + ".0.norm2"]); + auto to_q = std::dynamic_pointer_cast(blocks[p + ".0.to_q"]); + auto to_kv = std::dynamic_pointer_cast(blocks[p + ".0.to_kv"]); + auto to_out = std::dynamic_pointer_cast(blocks[p + ".0.to_out"]); + + ggml_tensor* xn = norm1->forward(ctx, x); + ggml_tensor* ln = norm2->forward(ctx, latents); + ggml_tensor* q = to_q->forward(ctx, ln); + ggml_tensor* kv_in = ggml_concat(ctx->ggml_ctx, xn, ln, 1); + ggml_tensor* kv = to_kv->forward(ctx, kv_in); + int64_t L = kv->ne[1]; + ggml_tensor* k = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0)); + ggml_tensor* v = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0])); + ggml_tensor* attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false); + attn = to_out->forward(ctx, attn); + latents = ggml_add(ctx->ggml_ctx, latents, attn); + + auto ff_norm = std::dynamic_pointer_cast(blocks[p + ".1.0"]); + auto ff_fc1 = std::dynamic_pointer_cast(blocks[p + ".1.1"]); + auto ff_fc2 = std::dynamic_pointer_cast(blocks[p + ".1.3"]); + ggml_tensor* h = ff_norm->forward(ctx, latents); + h = ff_fc1->forward(ctx, h); + h = ggml_gelu_erf(ctx->ggml_ctx, h); + h = ff_fc2->forward(ctx, h); + latents = ggml_add(ctx->ggml_ctx, latents, h); + } + + latents = proj_out->forward(ctx, latents); + latents = norm_out->forward(ctx, latents); + return latents; + } + }; + struct IPAdapterRunner : public GGMLRunner { ImageProjModel image_proj; + Resampler resampler; + bool is_plus = false; int64_t num_tokens = 4; std::string prefix; @@ -41,21 +125,54 @@ namespace IPAdapter { const std::string prefix, std::shared_ptr weight_manager = nullptr) : GGMLRunner(backend, weight_manager), prefix(prefix) { - int64_t ctx_dim = 768; - int64_t clip_dim = 1024; - int64_t out_dim = 3072; - auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight"); - if (norm_iter != tensor_storage_map.end()) { - ctx_dim = norm_iter->second.ne[0]; + is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end(); + if (is_plus) { + int64_t dim = 1280; + int64_t num_queries = 16; + int64_t embed_dim = 1280; + int64_t output_dim = 2048; + int64_t ff_inner = 5120; + auto latents_iter = tensor_storage_map.find(prefix + ".image_proj.latents"); + if (latents_iter != tensor_storage_map.end()) { + dim = latents_iter->second.ne[0]; + num_queries = latents_iter->second.ne[1]; + } + auto proj_in_iter = tensor_storage_map.find(prefix + ".image_proj.proj_in.weight"); + if (proj_in_iter != tensor_storage_map.end()) { + embed_dim = proj_in_iter->second.ne[0]; + } + auto proj_out_iter = tensor_storage_map.find(prefix + ".image_proj.proj_out.weight"); + if (proj_out_iter != tensor_storage_map.end()) { + output_dim = proj_out_iter->second.ne[1]; + } + auto ff_iter = tensor_storage_map.find(prefix + ".image_proj.layers.0.1.1.weight"); + if (ff_iter != tensor_storage_map.end()) { + ff_inner = ff_iter->second.ne[1]; + } + int64_t depth = 0; + while (tensor_storage_map.find(prefix + ".image_proj.layers." + std::to_string(depth) + ".0.to_q.weight") != tensor_storage_map.end()) { + depth++; + } + num_tokens = num_queries; + resampler = Resampler(dim, depth, num_queries, embed_dim, output_dim, ff_inner); + resampler.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); + } else { + int64_t ctx_dim = 768; + int64_t clip_dim = 1024; + int64_t out_dim = 3072; + auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight"); + if (norm_iter != tensor_storage_map.end()) { + ctx_dim = norm_iter->second.ne[0]; + } + auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight"); + if (proj_iter != tensor_storage_map.end()) { + clip_dim = proj_iter->second.ne[0]; + out_dim = proj_iter->second.ne[1]; + } + num_tokens = out_dim / ctx_dim; + image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim); + image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); } - auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight"); - if (proj_iter != tensor_storage_map.end()) { - clip_dim = proj_iter->second.ne[0]; - out_dim = proj_iter->second.ne[1]; - } - num_tokens = out_dim / ctx_dim; - image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim); - image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); } std::string get_desc() override { @@ -63,14 +180,18 @@ namespace IPAdapter { } void get_param_tensors(std::map& tensors, const std::string = "") { - image_proj.get_param_tensors(tensors, prefix + ".image_proj"); + if (is_plus) { + resampler.get_param_tensors(tensors, prefix + ".image_proj"); + } else { + image_proj.get_param_tensors(tensors, prefix + ".image_proj"); + } } ggml_cgraph* build_graph(const sd::Tensor& image_embeds_tensor) { ggml_cgraph* gf = new_graph_custom(1024); ggml_tensor* embeds = make_input(image_embeds_tensor); auto runner_ctx = get_context(); - ggml_tensor* out = image_proj.forward(&runner_ctx, embeds); + ggml_tensor* out = is_plus ? resampler.forward(&runner_ctx, embeds) : image_proj.forward(&runner_ctx, embeds); ggml_build_forward_expand(gf, out); return gf; } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index de3088178..47a980e61 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -2131,7 +2131,9 @@ class StableDiffusionGGML { return; } auto image_tensor = sd_image_to_tensor(image); - auto embed = get_clip_vision_output(image_tensor, true, -1); + auto embed = ip_adapter->is_plus + ? get_clip_vision_output(image_tensor, false, 2) + : get_clip_vision_output(image_tensor, true, -1); if (embed.empty()) { return; }