Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: in libHelper qnn setSeqLength->setCurSequenceLength #227

Merged
merged 40 commits into from
Jan 18, 2025
Merged

fix: in libHelper qnn setSeqLength->setCurSequenceLength #227

Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
e428bb5
dev: qnn multi input inference developing
oreomaker Nov 8, 2024
fca8584
Merge branch 'main' into develop-zh
oreomaker Nov 12, 2024
f94937d
feat: qnn multi chunk prefilling in new frontend
oreomaker Nov 13, 2024
83fefd3
Merge branch 'main' into develop-zh
yirongjie Nov 14, 2024
5ff02cd
fix: clearCache in RoPE
yirongjie Nov 14, 2024
68ddb99
fix: genarate with Padding
yirongjie Nov 14, 2024
28811fe
feat: make chunk_size configurable in demo_phonelm_npu
oreomaker Nov 14, 2024
d930d14
Merge branch 'main' into develop-zh
oreomaker Nov 16, 2024
9406c02
chore: andoid submodule diff
oreomaker Nov 16, 2024
caf4f9c
fix: qwen npu demos execution type set
oreomaker Nov 18, 2024
dfac9d5
feat: KVCacheNPU GQA
oreomaker Nov 18, 2024
2b20472
refactor: simplify kvcache npu
oreomaker Nov 21, 2024
fb95bcd
fix: kvcache npu seq
oreomaker Nov 21, 2024
865dcd6
Merge branch 'main' into develop-zh
oreomaker Nov 27, 2024
3ec8d77
chore: android diff
oreomaker Nov 27, 2024
3b0042d
dev: pipeline class init
oreomaker Dec 4, 2024
05bcd77
refactor: configurable chunk_size in qwen
oreomaker Dec 4, 2024
c3a0c4e
chore: remove qnn getBuildId
oreomaker Dec 7, 2024
9a56c4c
chore: clean qnn backend include
oreomaker Dec 7, 2024
f627381
dev: qnn new frontend pipeline(wrap implement)
oreomaker Dec 7, 2024
bec0a63
fix: kvcache nrep and stage switching bug in old frontend
oreomaker Dec 11, 2024
38fff58
feat: new frontend pipeline
oreomaker Jan 6, 2025
8d07588
chore: qnn qwen executable change
oreomaker Jan 6, 2025
693700b
feat: qnn prefill optimization, only do 1 seq lm_head
oreomaker Jan 7, 2025
be8258f
refactor: main qwen npu token post process
oreomaker Jan 7, 2025
f8def41
Merge branch 'main' into develop-zh
oreomaker Jan 7, 2025
7e20e8b
fix: qnn old frontend modeling backend assign
oreomaker Jan 7, 2025
1d526c1
fix: qnn total length and cur length conflict
oreomaker Jan 7, 2025
6070a4a
feat: configurable chunk size for HeadLinear
oreomaker Jan 7, 2025
a459c74
Update CMakeLists.txt
yirongjie Jan 8, 2025
0f03af2
fix:
yirongjie Jan 8, 2025
79151f2
fix
yirongjie Jan 8, 2025
c5973f1
feat: add gemma 2 model
oreomaker Jan 10, 2025
c19a0ad
chore: update gemma2 vocab
oreomaker Jan 10, 2025
2f0c3bf
Merge branch 'main' into develop-zh
yirongjie Jan 10, 2025
249ab14
doc: README
yirongjie Jan 10, 2025
04ceeab
fix: qnn demo libHelper setSeqLength
oreomaker Jan 17, 2025
2c8206c
Merge branch 'main' into develop-zh
yirongjie Jan 18, 2025
2baf222
Merge branch 'main' into develop-zh
oreomaker Jan 18, 2025
ebd050e
Merge branch 'develop-zh' of github.com:liang1232018/mllm into develo…
oreomaker Jan 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions tools/jni/LibHelper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ bool LibHelper::setUp(const std::string &base_path, std::string weights_path, st
return true;
});
Module::isFirstChunk = false;
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setCurSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setExecutionType(PROMPT);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->toggleSwitching();
Module::isMultiChunkPrefilling = true;
Expand Down Expand Up @@ -146,7 +146,7 @@ bool LibHelper::setUp(const std::string &base_path, std::string weights_path, st
return true;
});
Module::isFirstChunk = false;
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setCurSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setExecutionType(PROMPT);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->toggleSwitching();
Module::isMultiChunkPrefilling = true;
Expand Down Expand Up @@ -186,6 +186,11 @@ void LibHelper::run(std::string &input_str, uint8_t *image, unsigned max_step, u
const int chunk_num = seq_length_padding / chunk_size;
bool isSwitched = false;

// set total seq length for HeadLinear execute, which can not get the real seq length from Opts
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setTotalSequenceLength(real_seq_length);
// set chunk size for the HeadLinear execute, which can not get the chunk size from Opts
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setChunkSize(chunk_size);

LlmTextGeneratorOpts opt{
.max_new_tokens = 1,
.do_sample = false,
Expand Down Expand Up @@ -226,7 +231,7 @@ void LibHelper::run(std::string &input_str, uint8_t *image, unsigned max_step, u
});
Module::isFirstChunk = false;
}
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setSequenceLength(real_seq_length);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setCurSequenceLength(real_seq_length);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setExecutionType(AUTOREGRESSIVE);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->toggleSwitching();

Expand Down Expand Up @@ -259,7 +264,7 @@ void LibHelper::run(std::string &input_str, uint8_t *image, unsigned max_step, u
if (!not_end) { return false; }
return true;
});
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setCurSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setExecutionType(PROMPT);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->toggleSwitching();
} else { // CPU
Expand Down Expand Up @@ -318,6 +323,11 @@ void LibHelper::run(std::string &input_str, uint8_t *image, unsigned max_step, u
const int chunk_num = seq_length_padding / chunk_size;
bool isSwitched = false;

// set total seq length for HeadLinear execute, which can not get the real seq length from Opts
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setTotalSequenceLength(real_seq_length);
// set chunk size for the HeadLinear execute, which can not get the chunk size from Opts
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setChunkSize(chunk_size);

LlmTextGeneratorOpts opt{
.max_new_tokens = 1,
.do_sample = false,
Expand Down Expand Up @@ -360,7 +370,7 @@ void LibHelper::run(std::string &input_str, uint8_t *image, unsigned max_step, u
});
Module::isFirstChunk = false;
}
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setSequenceLength(real_seq_length);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setCurSequenceLength(real_seq_length);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setExecutionType(AUTOREGRESSIVE);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->toggleSwitching();

Expand Down Expand Up @@ -393,7 +403,7 @@ void LibHelper::run(std::string &input_str, uint8_t *image, unsigned max_step, u
if (!not_end) { return false; }
return true;
});
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setCurSequenceLength(0);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->setExecutionType(PROMPT);
static_cast<CPUBackend *>(Backend::global_backends[MLLM_CPU])->toggleSwitching();
} else { // CPU
Expand Down
Loading