diff options
| author | Ziang Wu <97337387+ZiangWu-77@users.noreply.github.com> | 2024-03-28 22:33:10 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-03-28 16:33:10 +0200 |
| commit | 66ba56025602270152f5ba5234f3a80be3dee1c9 (patch) | |
| tree | fdfabda7a33561130fef9e43bece9bfd3a13ee84 /examples/llava/clip.cpp | |
| parent | 0308f5e3d7bf9879f818b1a4ae589ff36b242af5 (diff) | |
llava : fix MobileVLM (#6364)
* fix empty bug
* Update MobileVLM-README.md
added more results on devices
* Update MobileVLM-README.md
* Update MobileVLM-README.md
* Update MobileVLM-README.md
* Update MobileVLM-README.md
* Update MobileVLM-README.md
* Update MobileVLM-README.md
* Update examples/llava/MobileVLM-README.md
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
* Update MobileVLM-README.md
remove gguf links
---------
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
Diffstat (limited to 'examples/llava/clip.cpp')
| -rw-r--r-- | examples/llava/clip.cpp | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp index 40c97626..5954bf6c 100644 --- a/examples/llava/clip.cpp +++ b/examples/llava/clip.cpp @@ -835,9 +835,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0); // weight ne = [3, 3, 2048, 1] struct ggml_tensor * peg_0 = ggml_conv_depthwise_2d(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1); - peg_0 = ggml_add(ctx0, peg_0, mlp_2); peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3)); peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b); + mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 2, 0, 3)); + peg_0 = ggml_add(ctx0, peg_0, mlp_2); peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]); embeddings = peg_0; } @@ -1755,7 +1756,7 @@ int clip_n_patches(const struct clip_ctx * ctx) { int n_patches = (params.image_size / params.patch_size) * (params.image_size / params.patch_size); - if (ctx->proj_type == PROJECTOR_TYPE_LDP) { + if (ctx->proj_type == PROJECTOR_TYPE_LDP || ctx->proj_type == PROJECTOR_TYPE_LDPV2) { n_patches /= 4; } |