mirror of
https://github.com/Dynamis-Labs/spectralquant.git
synced 2026-10-19 00:39:46 -05:00
Ashwin Gopinath
8d5a431276
Complete repository for "3% Is All You Need: Breaking TurboQuant's Compression Limit via Spectral Structure" — submitted to arXiv. Includes source code, experiment scripts, raw results, paper (LaTeX + PDF), figures, tests, and full reproduction instructions.
221 lines
5.4 KiB
JSON
221 lines
5.4 KiB
JSON
{
|
|
"Qwen2.5-1.5B-Instruct": {
|
|
"model": "Qwen2.5-1.5B-Instruct",
|
|
"n_layers": 28,
|
|
"n_kv_heads": 2,
|
|
"head_dim": 128,
|
|
"d_eff": 4,
|
|
"kappa": 1.247033076626914,
|
|
"configs": {
|
|
"TQ_3bit": {
|
|
"cos_sim": 0.8442779799799124,
|
|
"ratio": 5.019607843137255,
|
|
"avg_bits": 3.1875,
|
|
"mb_8k": 46.792704,
|
|
"n": 300
|
|
},
|
|
"TQ_2bit": {
|
|
"cos_sim": 0.7896283919115862,
|
|
"ratio": 7.314285714285714,
|
|
"avg_bits": 2.1875,
|
|
"mb_8k": 32.11264,
|
|
"n": 300
|
|
},
|
|
"SQ_selQJL": {
|
|
"cos_sim": 0.8476780078808467,
|
|
"ratio": 5.91907514450867,
|
|
"avg_bits": 2.703125,
|
|
"mb_8k": 39.682048,
|
|
"n": 300
|
|
},
|
|
"SQ_selQJL_v2tail": {
|
|
"cos_sim": 0.8342963253458341,
|
|
"ratio": 7.211267605633803,
|
|
"avg_bits": 2.21875,
|
|
"mb_8k": 32.571392,
|
|
"n": 300
|
|
},
|
|
"SQ_selQJL_v1tail": {
|
|
"cos_sim": 0.7834125710527102,
|
|
"ratio": 9.225225225225225,
|
|
"avg_bits": 1.734375,
|
|
"mb_8k": 25.460736,
|
|
"n": 300
|
|
},
|
|
"SQ_noQJL_v3": {
|
|
"cos_sim": 0.8615142812828223,
|
|
"ratio": 5.953488372093023,
|
|
"avg_bits": 2.6875,
|
|
"mb_8k": 39.452672,
|
|
"n": 300
|
|
},
|
|
"SQ_noQJL_v2tail": {
|
|
"cos_sim": 0.8500166252752145,
|
|
"ratio": 7.26241134751773,
|
|
"avg_bits": 2.203125,
|
|
"mb_8k": 32.342016,
|
|
"n": 300
|
|
},
|
|
"SQ_noQJL_v1tail": {
|
|
"cos_sim": 0.8024012971669435,
|
|
"ratio": 9.309090909090909,
|
|
"avg_bits": 1.71875,
|
|
"mb_8k": 25.23136,
|
|
"n": 300
|
|
},
|
|
"SQ_k1_noQJL_v2t": {
|
|
"cos_sim": 0.7690331786870956,
|
|
"ratio": 13.2987012987013,
|
|
"avg_bits": 1.203125,
|
|
"mb_8k": 17.661952,
|
|
"n": 300
|
|
}
|
|
}
|
|
},
|
|
"Qwen2.5-7B-Instruct": {
|
|
"model": "Qwen2.5-7B-Instruct",
|
|
"n_layers": 28,
|
|
"n_kv_heads": 4,
|
|
"head_dim": 128,
|
|
"d_eff": 4,
|
|
"kappa": 1.2375758588314056,
|
|
"configs": {
|
|
"TQ_3bit": {
|
|
"cos_sim": 0.8538468626514077,
|
|
"ratio": 5.019607843137255,
|
|
"avg_bits": 3.1875,
|
|
"mb_8k": 93.585408,
|
|
"n": 600
|
|
},
|
|
"TQ_2bit": {
|
|
"cos_sim": 0.7760489912455281,
|
|
"ratio": 7.314285714285714,
|
|
"avg_bits": 2.1875,
|
|
"mb_8k": 64.22528,
|
|
"n": 600
|
|
},
|
|
"SQ_selQJL": {
|
|
"cos_sim": 0.854534007711336,
|
|
"ratio": 5.91907514450867,
|
|
"avg_bits": 2.703125,
|
|
"mb_8k": 79.364096,
|
|
"n": 600
|
|
},
|
|
"SQ_selQJL_v2tail": {
|
|
"cos_sim": 0.8399755799459915,
|
|
"ratio": 7.211267605633803,
|
|
"avg_bits": 2.21875,
|
|
"mb_8k": 65.142784,
|
|
"n": 600
|
|
},
|
|
"SQ_selQJL_v1tail": {
|
|
"cos_sim": 0.7859353394682209,
|
|
"ratio": 9.225225225225225,
|
|
"avg_bits": 1.734375,
|
|
"mb_8k": 50.921472,
|
|
"n": 600
|
|
},
|
|
"SQ_noQJL_v3": {
|
|
"cos_sim": 0.8815342168261607,
|
|
"ratio": 5.953488372093023,
|
|
"avg_bits": 2.6875,
|
|
"mb_8k": 78.905344,
|
|
"n": 600
|
|
},
|
|
"SQ_noQJL_v2tail": {
|
|
"cos_sim": 0.8681940499320626,
|
|
"ratio": 7.26241134751773,
|
|
"avg_bits": 2.203125,
|
|
"mb_8k": 64.684032,
|
|
"n": 600
|
|
},
|
|
"SQ_noQJL_v1tail": {
|
|
"cos_sim": 0.8168674294153849,
|
|
"ratio": 9.309090909090909,
|
|
"avg_bits": 1.71875,
|
|
"mb_8k": 50.46272,
|
|
"n": 600
|
|
},
|
|
"SQ_k1_noQJL_v2t": {
|
|
"cos_sim": 0.7843631104379892,
|
|
"ratio": 13.2987012987013,
|
|
"avg_bits": 1.203125,
|
|
"mb_8k": 35.323904,
|
|
"n": 600
|
|
}
|
|
}
|
|
},
|
|
"Qwen2.5-14B-Instruct": {
|
|
"model": "Qwen2.5-14B-Instruct",
|
|
"n_layers": 48,
|
|
"n_kv_heads": 8,
|
|
"head_dim": 128,
|
|
"d_eff": 4,
|
|
"kappa": 1.2199700356771548,
|
|
"configs": {
|
|
"TQ_3bit": {
|
|
"cos_sim": 0.9226097098489603,
|
|
"ratio": 5.019607843137255,
|
|
"avg_bits": 3.1875,
|
|
"mb_8k": 320.864256,
|
|
"n": 1200
|
|
},
|
|
"TQ_2bit": {
|
|
"cos_sim": 0.8248648208379745,
|
|
"ratio": 7.314285714285714,
|
|
"avg_bits": 2.1875,
|
|
"mb_8k": 220.20096,
|
|
"n": 1200
|
|
},
|
|
"SQ_selQJL": {
|
|
"cos_sim": 0.9146691088626782,
|
|
"ratio": 5.91907514450867,
|
|
"avg_bits": 2.703125,
|
|
"mb_8k": 272.105472,
|
|
"n": 1200
|
|
},
|
|
"SQ_selQJL_v2tail": {
|
|
"cos_sim": 0.8986268473664919,
|
|
"ratio": 7.211267605633803,
|
|
"avg_bits": 2.21875,
|
|
"mb_8k": 223.346688,
|
|
"n": 1200
|
|
},
|
|
"SQ_selQJL_v1tail": {
|
|
"cos_sim": 0.8371809188773235,
|
|
"ratio": 9.225225225225225,
|
|
"avg_bits": 1.734375,
|
|
"mb_8k": 174.587904,
|
|
"n": 1200
|
|
},
|
|
"SQ_noQJL_v3": {
|
|
"cos_sim": 0.9484994028011958,
|
|
"ratio": 5.953488372093023,
|
|
"avg_bits": 2.6875,
|
|
"mb_8k": 270.532608,
|
|
"n": 1200
|
|
},
|
|
"SQ_noQJL_v2tail": {
|
|
"cos_sim": 0.9334064636131127,
|
|
"ratio": 7.26241134751773,
|
|
"avg_bits": 2.203125,
|
|
"mb_8k": 221.773824,
|
|
"n": 1200
|
|
},
|
|
"SQ_noQJL_v1tail": {
|
|
"cos_sim": 0.8744549995660782,
|
|
"ratio": 9.309090909090909,
|
|
"avg_bits": 1.71875,
|
|
"mb_8k": 173.01504,
|
|
"n": 1200
|
|
},
|
|
"SQ_k1_noQJL_v2t": {
|
|
"cos_sim": 0.8246567923823992,
|
|
"ratio": 13.2987012987013,
|
|
"avg_bits": 1.203125,
|
|
"mb_8k": 121.110528,
|
|
"n": 1200
|
|
}
|
|
}
|
|
}
|
|
} |