Update readme for gpu kernels

Add falcon-e support

README.md

@@ -137,6 +137,20 @@ This project is based on the [llama.cpp](https://github.com/ggerganov/llama.cpp)
         <td>&#9989;</td>
         <td>&#10060;</td>
     </tr>
+    <tr>
+        <td rowspan="2"><a href="https://huggingface.co/collections/tiiuae/falcon-edge-series-6804fd13344d6d8a8fa71130">Falcon-E Family</a></td>
+        <td rowspan="2">1B-3B</td>
+        <td>x86</td>
+        <td>&#9989;</td>
+        <td>&#10060;</td>
+        <td>&#9989;</td>
+    </tr>
+    <tr>
+        <td>ARM</td>
+        <td>&#9989;</td>
+        <td>&#9989;</td>
+        <td>&#10060;</td>
+    </tr>
 </table>
 
 

gpu/README.md

@@ -73,7 +73,9 @@ It significantly improves GEMV throughput when processing quantized weights and
 
 ## Performance
 
-Kernel performance (tested on NVIDIA A100 40GB GPU):
+### Kernel Benchmarks
+
+Tested on NVIDIA A100 40GB GPU, our custom W2A8 kernel shows significant speedups over standard BF16 implementations:
 
 | Shape (N×K)         | W2A8 Latency (us) | BF16 Latency (us) | Speedup Ratio        |
 |---------------------|-------------------|-------------------|----------------------|
@@ -86,8 +88,20 @@ Kernel performance (tested on NVIDIA A100 40GB GPU):
 | 3200 × 10240        | 19.64             | 60.79             |   3.10               |
 | 20480 × 3200        | 30.99             | 112.39            |   3.63               |
 
-Generation throughput:
+### End-to-End Generation Latency
 
-| BF16 (tokens/s) | W2A8 (tokens/s) | Speedup Ratio |
-|---|---|---|
-| 10.9 | 213.3 | 19.6 |
+Compared to a similarly-sized BF16 model (Gemma-2-2B using vLLM), BitNet-b1.58-2B with our kernel achieves consistent speedups across workloads:
+
+| Input Length | Output Length | BF16 Latency (ms) | W2A8 Latency (ms) | Speedup Ratio |
+| --- | --- | --- | --- | --- |
+| 64 | 16 | 187.64 | 57.40 | 3.27 |
+| 64 | 32 | 353.50 | 112.22 | 3.15 |
+| 64 | 64 | 683.23 | 221.08 | 3.09 |
+| 256 | 16 | 183.14 | 61.24 | 2.99 |
+| 256 | 32 | 353.14 | 115.47 | 3.06 |
+| 256 | 64 | 684.24 | 224.16 | 3.05 |
+| 512 | 16 | 208.99 | 68.06 | 3.07 |
+| 512 | 32 | 354.33 | 122.72 | 2.89 |
+| 512 | 64 | 709.65 | 231.82 | 3.06 |
+
+*Note: Comparison uses equivalent-sized models (2B parameters) on NVIDIA A100 40GB GPU.*

setup_env.py

@@ -44,6 +44,18 @@ SUPPORTED_HF_MODELS = {
     "microsoft/BitNet-b1.58-2B-4T": {
         "model_name": "BitNet-b1.58-2B-4T",
     },
+    "tiiuae/Falcon-E-3B-Instruct": {
+        "model_name": "Falcon-E-3B-Instruct",
+    },
+    "tiiuae/Falcon-E-1B-Instruct": {
+        "model_name": "Falcon-E-1B-Instruct",
+    },
+    "tiiuae/Falcon-E-3B-Base": {
+        "model_name": "Falcon-E-3B-Base",
+    },
+    "tiiuae/Falcon-E-1B-Base": {
+        "model_name": "Falcon-E-1B-Base",
+    },
 }
 
 SUPPORTED_QUANT_TYPES = {
@@ -144,7 +156,7 @@ def setup_gguf():
 def gen_code():
     _, arch = system_info()
     
-    llama3_f3_models = set([model['model_name'] for model in SUPPORTED_HF_MODELS.values() if model['model_name'].startswith("Falcon3") or model['model_name'].startswith("Llama")])
+    llama3_f3_models = set([model['model_name'] for model in SUPPORTED_HF_MODELS.values() if model['model_name'].startswith("Falcon") or model['model_name'].startswith("Llama")])
 
     if arch == "arm64":
         if args.use_pretuned:

utils/convert-hf-to-gguf-bitnet.py

@@ -319,6 +319,9 @@ class Model(ABC):
         if chkhsh == "8aeee3860c56296a157a1fe2fad249ec40aa59b1bb5709f4ade11c4e6fe652ed":
             # ref: https://huggingface.co/tiiuae/falcon-7b
             res = "falcon"
+        if chkhsh == "a6b57017d60e6edb4d88ecc2845188e0eb333a70357e45dcc9b53964a73bbae6":
+            # ref: https://huggingface.co/tiiuae/Falcon-E-3B-Instruct
+            res = "falcon_e"
         if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f":
             # ref: https://huggingface.co/BAAI/bge-small-en-v1.5
             res = "bert-bge"