doc-to-lora/tmp/test_needle_haystack_plot.py

#!/usr/bin/env python3
"""
Test script to create a line plot showing needle-in-a-haystack task accuracy vs input length.
"""

import matplotlib as mpl
import matplotlib.pyplot as plt

latte_style = "https://raw.githubusercontent.com/51616/catppuccin-matplotlib/main/src/mplcatppuccin/data/latte.mplstyle"
plt.style.use(["ggplot", latte_style])
plt.rcParams["axes.prop_cycle"] = mpl.cycler(
    color=[
        "#FFB83D",
        "#5571EA",
    ]
)
plt.rcParams["axes.facecolor"] = "white"
plt.rcParams["figure.facecolor"] = "white"
plt.rcParams["savefig.facecolor"] = "white"

# plt.rcParams["font.family"] = "Ubuntu"
# plt.rcParams["font.size"] = 14
# plt.rcParams["font.weight"] = "bold"
plt.rcParams["axes.labelweight"] = "bold"
plt.rcParams["axes.facecolor"] = "F7FBFC"
plt.rcParams["figure.facecolor"] = "white"
plt.rcParams["savefig.facecolor"] = "white"
plt.rcParams["grid.color"] = "cccccc"
plt.rcParams["grid.linewidth"] = 1
plt.rcParams["axes.edgecolor"] = "ccd0da"
plt.rcParams["legend.facecolor"] = "white"
plt.rcParams["legend.fontsize"] = 16
plt.rcParams["xtick.labelsize"] = 16
plt.rcParams["ytick.labelsize"] = 16


def main():
    # Token bins from the user's specification
    tok_bins = [1024 * (i + 1) for i in range(16)]
    tok_bins += [2**14 + 2**12 * (i + 1) for i in range(4)]
    tok_bins += [2**15 + 2**13 * (i + 1) for i in range(12)]
    x_values = tok_bins
    print(x_values)

    y_values = {
        "Base model w/ context": [
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            0.99,
            0.80,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
            0.00,
        ],
        "Ours": [
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            1.00,
            0.95,
            0.87,
            0.80,
            0.65,
            0.53,
            0.43,
            0.37,
            0.28,
            0.24,
            0.17,
            0.15,
        ],
    }

    # Create the plot
    cleaner_x_vals = [
        2**10,
        2**11,
        2**12,
        2**13,
        2**13 + 2**10,
        2**13 + 2**11,
        2**14,
        2**15,
        2**15 + 2**13,
    ]
    plt.figure(figsize=(9, 6))
    for name, ys in y_values.items():
        xys = [
            (x, y)
            for x, y in zip(x_values, ys)
            if x in cleaner_x_vals or x > 2**15 + 2**13
        ]
        plt.plot(
            [xy[0] for xy in xys],
            [xy[1] for xy in xys],
            marker="o",
            linewidth=2,
            markersize=8,
            alpha=1.0,
            label=name,
            markeredgecolor="black",
            markeredgewidth=1,
            markerfacecolor=None,
        )

    # Customize the plot
    plt.xlabel("Haystack Length (tokens)", fontsize=20)
    plt.ylabel("ROUGE-L F1 Score", fontsize=20)
    plt.title(
        "NIAH Retrieval Performance",
        fontsize=24,
        fontweight="bold",
    )
    plt.grid(True, alpha=0.3)

    # Set log scale for x-axis
    plt.xscale("log", base=2)

    # Format x-axis labels with K suffix (representing 1024 multiples)
    # x_labels = [
    #     f"{x // 1024}K" if x >= 1024 else str(x)
    #     for x in x_values
    #     if x in cleaner_x_vals or x > 2**15 + 2**13
    # ]
    # plt.xticks(
    #     [x for x in x_values if x in cleaner_x_vals or x > 2**15 + 2**13],
    #     x_labels,
    #     rotation=90,
    #     fontsize=8,
    # )

    # Remove minor ticks for cleaner appearance
    plt.gca().xaxis.set_minor_locator(plt.NullLocator())

    # Add some styling
    plt.ylim(-0.05, 1.05)
    plt.xlim(min(x_values) * 0.8, max(x_values) * 1.2)

    # Add horizontal lines for reference

    # Add legend
    plt.legend(loc="lower left")

    # Tight layout for better appearance
    plt.tight_layout()

    # Save the plot
    plt.savefig(
        "/home/tan/research/ctx-to-lora/tmp/needle_haystack_accuracy_plot.pdf",
        format="pdf",
        dpi=300,
        bbox_inches="tight",
    )

    # Show statistics
    print(f"Generated plot with {len(x_values)} data points")
    print(f"Input length range: {min(x_values):.0f} - {max(x_values):.0f} tokens")
    print(
        f"Plot saved to: /home/tan/research/ctx-to-lora/tmp/needle_haystack_accuracy_plot.pdf"
    )

    # Display the plot
    plt.show()


if __name__ == "__main__":
    main()