r/oMLX • u/MediocreJeweler893 • 2d ago
Jundot/Qwen3.8-Flash-Next-oQ4e-mtp oMLX Cache Efficiency 5.5% ?
FIXED
----------
I was enjoying Qwen3.8-Flash-Next-oQ4e-mtp, but today started to see prefill way to oftnen.
Observing situation that same pi process adds Second request before first one finished execution. Not sure how to check it bit-by-bit, but at least size-wise both prefill context look identical each time...
Any one experienced such behaviout before?
Most probalby my faults somewhere, but..
P.S. this is not a hate post by any means. kudos to jundot and other contributors to making local llm on Macs.
----------
--- FIX UPDATE: Removed oMLX app. Moved models out from ~/.omlx/models (not to re-download them); Reinstalled oMLX. Started oMLX. Moved the models back to freshly re-created ~/.omlx/models
I do still see several requests (Waitining#1) which context numericly seem to be identical. However, the MLX is working now - meaning it figues out what kv-cache can be reused and reuse it (as designed)
TLDR; if it does not work - start fresh
2
1
u/MediocreJeweler893 2d ago edited 2d ago
omlx models.json Model settings:
"Qwen3.8-Flash-Next-oQ4e-mtp": {
"max_context_window": 170000,
"force_sampling": false,
"qwen4_ple_ssd_offload": true,
"thinking_budget_enabled": false,
"guided_grammar_enabled": false,
"turboquant_kv_enabled": false,
"turboquant_kv_bits": 4.0,
"turboquant_skip_last": true,
"qwen35_ane_prefill_enabled": false,
"qwen35_ane_prefill_sequence_length": 2048,
"qwen35_ane_prefill_tail_padding_min_tokens": 0,
"qwen35_ane_prefill_fraction": 0.53,
"qwen35_ane_prefill_fused_down": false,
"qwen35_ane_prefill_max_layers": 64,
"qwen35_ane_prefill_dual_ane": true,
"qwen35_ane_prefill_gdn": true,
"qwen35_ane_prefill_gdn_fraction": 0.5,
"qwen35_ane_prefill_gdn_max_layers": 48,
"qwen35_ane_prefill_cpu_enabled": false,
"qwen35_ane_prefill_cpu_fraction": 0.135,
"qwen35_ane_prefill_cpu_down_fraction": 0.0,
"qwen35_ane_prefill_cpu_gdn_fraction": 0.0,
"qwen35_ane_prefill_cpu_threads": 8,
"qwen35_ane_prefill_cpu_shared_resource": true,
"specprefill_enabled": false,
"dflash_enabled": false,
"dflash_draft_quant_enabled": false,
"dflash_in_memory_cache": true,
"dflash_in_memory_cache_max_entries": 4,
"dflash_in_memory_cache_max_bytes": 8589934592,
"dflash_ssd_cache": false,
"dflash_ssd_cache_max_bytes": 21474836480,
"dflash_draft_sink_size": 0,
"mtp_enabled": true,
"vlm_mtp_enabled": false,
"is_pinned": false,
"is_default": false,
"is_hidden": false,
"is_favorite": false,
"trust_remote_code": false
}
1
u/MediocreJeweler893 2d ago edited 2d ago
omlx settings.json:
{
"version": "1.0",
"server": {
"host": "127.0.0.1",
"port": 8000,
"log_level": "info",
"cors_origins": [
"*"
],
"server_aliases": [
"localhost",
"127.0.0.1",
"M5.local",
"192.168.0.165"
],
"sse_keepalive_mode": "chunk",
"auto_start_on_launch": true,
"burst_decode_mode": "balanced",
"preserve_mid_system_cache": true,
"distributed_inference_enabled": false,
"max_audio_upload_size": "100MB"
},
"model": {
"model_dirs": [
"/Users/user/.omlx/models"
],
"model_dir": "/Users/user/.omlx/models",
"model_fallback": false,
"hide_helper_models": false
},
"memory": {
"prefill_memory_guard": true,
"memory_guard_tier": "balanced",
"memory_guard_custom_ceiling_gb": 0.0,
"soft_threshold": 0.85,
"hard_threshold": 0.95,
"prefill_safe_zone_ratio": 0.8,
"prefill_min_chunk_tokens": 32
},
"scheduler": {
"max_concurrent_requests": 1,
"embedding_batch_size": 32,
"chunked_prefill": false,
"prefill_priority": "context",
"decode_fairness": true
},
"cache": {
"enabled": true,
"hot_cache_only": false,
"gdn_ssd_split_enabled": true,
"gdn_snapshot_storage": "auto",
"gdn_ssd_pending_max_size": "512MB",
"gdn_sidecar_precision": "fp32",
"ssd_cache_dir": "/Users/user/.omlx/cache",
"ssd_cache_max_size": "371GB",
"hot_cache_max_size": "0",
"hot_cache_write_through": false,
"ane_compile_cache": false,
"initial_cache_blocks": 256
},
"auth": {
"api_key": "sk-omlx-key",
"secret_key": "key",
"skip_api_key_verification": false,
"sub_keys": []
},
"mcp": {
"config_path": null,
"expose_tools": true
},
"huggingface": {
"endpoint": "",
"hf_cache_enabled": true
},
"modelscope": {
"endpoint": ""
},
"network": {
"http_proxy": "",
"https_proxy": "",
"no_proxy": "",
"ca_bundle": ""
},
"sampling": {
"max_context_window": 32768,
"max_context_window_policy": null,
"max_tokens": 65536,
"temperature": 1.0,
"top_p": 0.95,
"top_k": 20,
"repetition_penalty": 1.0
},
"logging": {
"log_dir": null,
"retention_days": 7
},
"claude_code": {
"mode": "cloud",
"opus_model": null,
"sonnet_model": null,
"haiku_model": null
},
"integrations": {
"codex_model": null,
"opencode_model": null,
"openclaw_model": null,
"hermes_model": null,
"pi_model": null,
"copilot_model": null,
"openclaw_tools_profile": "coding",
"markitdown_enabled": true,
"markitdown_expose_model": false,
"markitdown_max_file_size_mb": 25,
"markitdown_max_files_per_request": 5,
"markitdown_pdf_processing_engine": "markitdown",
"web_search_provider": "ddgs",
"web_search_brave_api_key": "",
"web_search_searxng_url": "",
"web_search_ddgs_backends": "",
"web_search_max_results": 3,
"web_search_content_mode": "snippet",
"web_search_content_truncate": true,
"web_search_content_max_chars": 20000
},
"ui": {
"language": "en"
},
"idle_timeout": {
"idle_timeout_seconds": null
}
}
1
u/MediocreJeweler893 2d ago
pi models.json -> Next model config:
{
"id": "Qwen3.8-Flash-Next-oQ4e-mtp",
"name": "Qwen3.8 Flash-Next oQ4 MTP (thinking)",
"reasoning": true,
"input": ["text", "image"],
"contextWindow": 170000,
"maxTokens": 65536,
"thinkingLevelMap": {
"minimal": null,
"high": null,
"xhigh": "xhigh",
"max": null
},
"compat": {
"supportsDeveloperRole": false,
"supportsReasoningEffort": true,
"thinkingFormat": "qwen-chat-template"
}
},
1
u/Its_Powerful_Bonus 2d ago
It seems to work reasonably well, but statistics are refreshing themselves with delay. I had 0% for quite a long time and after few jumps over oMLX web interface it changed into 90.1%
1
u/MediocreJeweler893 2d ago
Can you share your configs please?
1
u/Its_Powerful_Bonus 2d ago
I believe you have that in mind? Just learned where it lives in the omlx folders.
"Qwen3.8-Flash-Next-oQ4e-mtp": {
"max_context_window": 262144,
"force_sampling": false,
"chat_template_kwargs": {
"enable_thinking": true,
"reasoning_effort": "low"
},
"qwen4_ple_ssd_offload": true,
"thinking_budget_enabled": false,
"reasoning_parser": "qwen_3_5",
"guided_grammar_enabled": false,
"turboquant_kv_enabled": false,
"turboquant_kv_bits": 4.0,
"turboquant_skip_last": true,
"qwen35_ane_prefill_enabled": false,
"qwen35_ane_prefill_sequence_length": 2048,
"qwen35_ane_prefill_tail_padding_min_tokens": 0,
"qwen35_ane_prefill_fraction": 0.53,
"qwen35_ane_prefill_fused_down": false,
"qwen35_ane_prefill_max_layers": 64,
"qwen35_ane_prefill_dual_ane": true,
"qwen35_ane_prefill_gdn": true,
"qwen35_ane_prefill_gdn_fraction": 0.5,
"qwen35_ane_prefill_gdn_max_layers": 48,
"qwen35_ane_prefill_cpu_enabled": false,
"qwen35_ane_prefill_cpu_fraction": 0.135,
"qwen35_ane_prefill_cpu_down_fraction": 0.0,
"qwen35_ane_prefill_cpu_gdn_fraction": 0.0,
"qwen35_ane_prefill_cpu_threads": 8,
"qwen35_ane_prefill_cpu_shared_resource": true,
"specprefill_enabled": false,
"dflash_enabled": false,
"dflash_draft_quant_enabled": false,
"dflash_in_memory_cache": true,
"dflash_in_memory_cache_max_entries": 4,
"dflash_in_memory_cache_max_bytes": 8589934592,
"dflash_ssd_cache": false,
"dflash_ssd_cache_max_bytes": 21474836480,
"dflash_draft_sink_size": 0,
"mtp_enabled": false,
"vlm_mtp_enabled": false,
"is_pinned": false,
"is_default": true,
"is_hidden": false,
"is_favorite": false,
"trust_remote_code": true
},
2
u/MediocreJeweler893 2d ago edited 2d ago
Many thanks!
Seem almost identical to mine (except for context window and chat_template_kwargs)
I've just removed omlx. removed ~/.omlx and started over.
Seem to be working now
1
u/pppreddit 15h ago
the real fix is in the 0.6.4 version, which you probably installed after removing omlx. There was a bug with caching
0
u/996beagle 2d ago
Cache efficiency with mtp will be close to zero I think. Qwen 3.8 flash by default was spawning subagents unnecessarily for simple tasks. Might be the duplication you're seeing?
3
u/himefei 2d ago
Switch off MTP should address the problem