sqlite-vec/examples/nbc-headlines/3_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table>\n",
       "<thead>\n",
       "<tr style=\"text-align: center;\">\n",
       "<th>\n",
       "vec_version()\n",
       "</th>\n",
       "<th>\n",
       "lembed_version()\n",
       "</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td style=\"text-align: left;\">\n",
       "v0.1.3-alpha.2\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "v0.0.1-alpha.7\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "<div style=\"text-align: right;\">\n",
       "1 row × 2 columns\n",
       "</div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "\u001b[0m┌\u001b[0m\u001b[0m────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m──────────────────\u001b[0m\u001b[0m┐\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mvec_version()\u001b[0m \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mlembed_version()\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m├\u001b[0m\u001b[0m────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m──────────────────\u001b[0m\u001b[0m┤\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mv0.1.3-alpha.2\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mv0.0.1-alpha.7  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m└\u001b[0m\u001b[0m────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m──────────────────\u001b[0m\u001b[0m┘\n",
       "\u001b[0m\u001b[0m"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".open tmp3.db\n",
    ".param set foo bar\n",
    "\n",
    ".load ../../dist/vec0\n",
    ".load ./lembed0\n",
    "\n",
    "insert into lembed_models(name, model)\n",
    "  values (\n",
    "    'default',\n",
    "    lembed_model_from_file('all-MiniLM-L6-v2.e4ce9877.q8_0.gguf')\n",
    "  );\n",
    "\n",
    "select vec_version(), lembed_version();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FTS Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table>\n",
       "<thead>\n",
       "<tr style=\"text-align: center;\">\n",
       "<th>\n",
       "rowid\n",
       "</th>\n",
       "<th>\n",
       "headline_highlighted\n",
       "</th>\n",
       "<th>\n",
       "rank\n",
       "</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td >\n",
       "4666\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Kamala Harris visits &lt;b&gt;Planned&lt;/b&gt; &lt;b&gt;Parenthood&lt;/b&gt; clinic\n",
       "</td>\n",
       "<td >\n",
       "-18.9139950477264\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "6521\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Former Marine sentenced to 9 years in prison for firebombing &lt;b&gt;Planned&lt;/b&gt; &lt;b&gt;Parenthood&lt;/b&gt; clinic\n",
       "</td>\n",
       "<td >\n",
       "-14.807022703838651\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "<div style=\"text-align: right;\">\n",
       "2 rows × 3 columns\n",
       "</div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "\u001b[0m┌\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m──────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┐\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mrowid\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mheadline_highlighted\u001b[0m                                                                                \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mrank\u001b[0m               \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m├\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m──────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┤\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 4666\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mKamala Harris visits <b>Planned</b> <b>Parenthood</b> clinic                                        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  -18.9139950477264\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 6521\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mFormer Marine sentenced to 9 years in prison for firebombing <b>Planned</b> <b>Parenthood</b> clinic\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-14.807022703838651\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m└\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m──────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┘\n",
       "\u001b[0m\u001b[0m"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".param set query planned parenthood\n",
    "\n",
    "select\n",
    "  rowid,\n",
    "  highlight(fts_headlines, 0, '<b>', '</b>') as headline_highlighted,\n",
    "  rank\n",
    "from fts_headlines\n",
    "where headline match :query\n",
    "order by rank\n",
    "limit 10;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Vector Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table>\n",
       "<thead>\n",
       "<tr style=\"text-align: center;\">\n",
       "<th>\n",
       "article_id\n",
       "</th>\n",
       "<th>\n",
       "headline\n",
       "</th>\n",
       "<th>\n",
       "distance\n",
       "</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td >\n",
       "9475\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Inside a Gen Z campaign to shake up Congress\n",
       "</td>\n",
       "<td >\n",
       "1.0655490159988403\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "11236\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Bunnie XO and Jelly Roll announce plans to have a baby using IVF\n",
       "</td>\n",
       "<td >\n",
       "1.071580171585083\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "4110\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "First over-the-counter birth control pill hits store shelves\n",
       "</td>\n",
       "<td >\n",
       "1.0831280946731567\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "1808\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "More nonpregnant women are requesting abortion pills to have on hand\n",
       "</td>\n",
       "<td >\n",
       "1.0897283554077148\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "11154\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Meet the anti-abortion group using white coats and research to advance its cause\n",
       "</td>\n",
       "<td >\n",
       "1.089759349822998\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "5980\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Deathbed confession leads to bodies of mother and daughter killed 24 years ago\n",
       "</td>\n",
       "<td >\n",
       "1.092090368270874\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "3540\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Holding photos of their deceased children, parents lobby Congress to pass online safety legislation\n",
       "</td>\n",
       "<td >\n",
       "1.1028637886047363\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "4109\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Alabama fertility clinic at the center of IVF ruling is resuming some procedures\n",
       "</td>\n",
       "<td >\n",
       "1.103183627128601\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "4889\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Inside the organized crime rings plaguing Ulta, T.J. Maxx, Walgreens and other retailers\n",
       "</td>\n",
       "<td >\n",
       "1.1056489944458008\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "86\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Florida bars transgender people from changing the sex on their driver&#39;s licenses\n",
       "</td>\n",
       "<td >\n",
       "1.1073163747787476\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "<div style=\"text-align: right;\">\n",
       "10 rows × 3 columns\n",
       "</div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "\u001b[0m┌\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────────────\u001b[0m\u001b[0m┐\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1marticle_id\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mheadline\u001b[0m                                                                                           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mdistance\u001b[0m          \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m├\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────────────\u001b[0m\u001b[0m┤\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      9475\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mInside a Gen Z campaign to shake up Congress                                                       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m1.0655490159988403\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m     11236\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mBunnie XO and Jelly Roll announce plans to have a baby using IVF                                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 1.071580171585083\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      4110\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mFirst over-the-counter birth control pill hits store shelves                                       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m1.0831280946731567\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      1808\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mMore nonpregnant women are requesting abortion pills to have on hand                               \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m1.0897283554077148\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m     11154\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mMeet the anti-abortion group using white coats and research to advance its cause                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 1.089759349822998\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      5980\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mDeathbed confession leads to bodies of mother and daughter killed 24 years ago                     \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 1.092090368270874\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      3540\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mHolding photos of their deceased children, parents lobby Congress to pass online safety legislation\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m1.1028637886047363\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      4109\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAlabama fertility clinic at the center of IVF ruling is resuming some procedures                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 1.103183627128601\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      4889\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mInside the organized crime rings plaguing Ulta, T.J. Maxx, Walgreens and other retailers           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m1.1056489944458008\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        86\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mFlorida bars transgender people from changing the sex on their driver's licenses                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m1.1073163747787476\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m└\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────────────\u001b[0m\u001b[0m┘\n",
       "\u001b[0m\u001b[0m"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".param set query planned parenthood\n",
    "\n",
    "select\n",
    "  article_id,\n",
    "  articles.headline,\n",
    "  distance\n",
    "from vec_headlines\n",
    "left join articles on articles.rowid = vec_headlines.article_id\n",
    "where headline_embedding match lembed(:query)\n",
    "  and k = 10;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## keyword-first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table>\n",
       "<thead>\n",
       "<tr style=\"text-align: center;\">\n",
       "<th>\n",
       "id\n",
       "</th>\n",
       "<th>\n",
       "headline\n",
       "</th>\n",
       "<th>\n",
       "match_type\n",
       "</th>\n",
       "<th>\n",
       "article_id\n",
       "</th>\n",
       "<th>\n",
       "rank_number\n",
       "</th>\n",
       "<th>\n",
       "score\n",
       "</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td >\n",
       "10098\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Kamala Harris says abortion bans are creating &#39;a health care crisis&#39;\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "fts\n",
       "</td>\n",
       "<td >\n",
       "10098\n",
       "</td>\n",
       "<td >\n",
       "1\n",
       "</td>\n",
       "<td >\n",
       "-10.678829270936067\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "States with abortion bans saw birth control prescriptions fall post-Dobbs, study finds\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "fts\n",
       "</td>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td >\n",
       "2\n",
       "</td>\n",
       "<td >\n",
       "-10.016316725971112\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "2292\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Ohio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "fts\n",
       "</td>\n",
       "<td >\n",
       "2292\n",
       "</td>\n",
       "<td >\n",
       "3\n",
       "</td>\n",
       "<td >\n",
       "-9.7149595994016\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "452\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "64K women and girls became pregnant due to rape in states with abortion bans, study estimates\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "fts\n",
       "</td>\n",
       "<td >\n",
       "452\n",
       "</td>\n",
       "<td >\n",
       "4\n",
       "</td>\n",
       "<td >\n",
       "-9.163558569425538\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Abortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "fts\n",
       "</td>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td >\n",
       "5\n",
       "</td>\n",
       "<td >\n",
       "-9.163558569425538\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "11154\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Meet the anti-abortion group using white coats and research to advance its cause\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "11154\n",
       "</td>\n",
       "<td >\n",
       "1\n",
       "</td>\n",
       "<td >\n",
       "0.7592407464981079\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "3958\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Supreme Court signals it is likely to reject a challenge to abortion pill access\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "3958\n",
       "</td>\n",
       "<td >\n",
       "2\n",
       "</td>\n",
       "<td >\n",
       "0.8076152205467224\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "6011\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Supreme Court wrestles with abortion clash over emergency room treatment for pregnant women\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "6011\n",
       "</td>\n",
       "<td >\n",
       "3\n",
       "</td>\n",
       "<td >\n",
       "0.812921941280365\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "3933\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "The Supreme Court puts the GOP’s abortion dilemma back in the spotlight: From the Politics Desk\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "3933\n",
       "</td>\n",
       "<td >\n",
       "4\n",
       "</td>\n",
       "<td >\n",
       "0.8234137296676636\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "7381\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Where abortion rights could be on the ballot this fall: From the Politics Desk\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "7381\n",
       "</td>\n",
       "<td >\n",
       "5\n",
       "</td>\n",
       "<td >\n",
       "0.8250945210456848\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "6680\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "More young people choosing permanent sterilization after abortion restrictions\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "6680\n",
       "</td>\n",
       "<td >\n",
       "6\n",
       "</td>\n",
       "<td >\n",
       "0.832757294178009\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "States with abortion bans saw birth control prescriptions fall post-Dobbs, study finds\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td >\n",
       "7\n",
       "</td>\n",
       "<td >\n",
       "0.837793231010437\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Abortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td >\n",
       "8\n",
       "</td>\n",
       "<td >\n",
       "0.8381417989730835\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "2646\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Trump campaign scrambles over abortion ban report as Democrats seize the moment\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "2646\n",
       "</td>\n",
       "<td >\n",
       "9\n",
       "</td>\n",
       "<td >\n",
       "0.8460941910743713\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9447\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "At a clinic in Hialeah, immigrants wrestle with Florida&#39;s new abortion ban\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "vec\n",
       "</td>\n",
       "<td >\n",
       "9447\n",
       "</td>\n",
       "<td >\n",
       "10\n",
       "</td>\n",
       "<td >\n",
       "0.8477099537849426\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "<div style=\"text-align: right;\">\n",
       "15 rows × 6 columns\n",
       "</div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "\u001b[0m┌\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┐\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mid\u001b[0m   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mheadline\u001b[0m                                                                                              \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mmatch_type\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1marticle_id\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mrank_number\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mscore\u001b[0m              \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m├\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┤\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m10098\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mKamala Harris says abortion bans are creating 'a health care crisis'                                  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mfts       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m     10098\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          1\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-10.678829270936067\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mStates with abortion bans saw birth control prescriptions fall post-Dobbs, study finds                \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mfts       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          2\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-10.016316725971112\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 2292\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mOhio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mfts       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      2292\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          3\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m   -9.7149595994016\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  452\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m64K women and girls became pregnant due to rape in states with abortion bans, study estimates         \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mfts       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       452\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          4\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m -9.163558569425538\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAbortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mfts       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          5\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m -9.163558569425538\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m11154\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mMeet the anti-abortion group using white coats and research to advance its cause                      \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m     11154\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          1\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.7592407464981079\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 3958\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mSupreme Court signals it is likely to reject a challenge to abortion pill access                      \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      3958\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          2\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.8076152205467224\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 6011\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mSupreme Court wrestles with abortion clash over emergency room treatment for pregnant women           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      6011\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          3\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  0.812921941280365\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 3933\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mThe Supreme Court puts the GOP’s abortion dilemma back in the spotlight: From the Politics Desk       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      3933\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          4\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.8234137296676636\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 7381\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk                        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      7381\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          5\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.8250945210456848\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 6680\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mMore young people choosing permanent sterilization after abortion restrictions                        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      6680\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          6\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  0.832757294178009\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mStates with abortion bans saw birth control prescriptions fall post-Dobbs, study finds                \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          7\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  0.837793231010437\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAbortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          8\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.8381417989730835\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 2646\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mTrump campaign scrambles over abortion ban report as Democrats seize the moment                       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      2646\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m          9\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.8460941910743713\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9447\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAt a clinic in Hialeah, immigrants wrestle with Florida's new abortion ban                            \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mvec       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      9447\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m         10\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.8477099537849426\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m└\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┘\n",
       "\u001b[0m\u001b[0m"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".param set query abortion bans\n",
    "\n",
    "\n",
    ".param set weight_fts 1.0\n",
    ".param set weight_vec 1.0\n",
    ".param set rrf_k 60\n",
    ".param set k 10\n",
    "\n",
    "\n",
    "with vec_matches as (\n",
    "  select\n",
    "    article_id,\n",
    "    row_number() over (order by distance) as rank_number,\n",
    "    distance as score\n",
    "  from vec_headlines\n",
    "  where\n",
    "    headline_embedding match lembed(:query)\n",
    "    and k = :k\n",
    "  order by distance\n",
    "),\n",
    "fts_matches as (\n",
    "  select\n",
    "    rowid as article_id,\n",
    "    row_number() over (order by rank) as rank_number,\n",
    "    rank as score\n",
    "  from fts_headlines\n",
    "  where headline match :query\n",
    "  limit :k\n",
    "),\n",
    "combined as (\n",
    "  select 'fts' as match_type, * from fts_matches\n",
    "  union all\n",
    "  select 'vec' as match_type, * from vec_matches\n",
    "),\n",
    "final as (\n",
    "  select\n",
    "    articles.id,\n",
    "    articles.headline,\n",
    "    combined.*\n",
    "  from combined\n",
    "  left join articles on articles.rowid = combined.article_id\n",
    ")\n",
    "select * from final;\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FTS + Vector search: RFF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[no code]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".param set  name alex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table>\n",
       "<thead>\n",
       "<tr style=\"text-align: center;\">\n",
       "<th>\n",
       "id\n",
       "</th>\n",
       "<th>\n",
       "headline\n",
       "</th>\n",
       "<th>\n",
       "vec_rank\n",
       "</th>\n",
       "<th>\n",
       "fts_rank\n",
       "</th>\n",
       "<th>\n",
       "combined_rank\n",
       "</th>\n",
       "<th>\n",
       "vec_distance\n",
       "</th>\n",
       "<th>\n",
       "fts_score\n",
       "</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "States with abortion bans saw birth control prescriptions fall post-Dobbs, study finds\n",
       "</td>\n",
       "<td >\n",
       "7\n",
       "</td>\n",
       "<td >\n",
       "2\n",
       "</td>\n",
       "<td >\n",
       "0.031054405392392875\n",
       "</td>\n",
       "<td >\n",
       "0.837793231010437\n",
       "</td>\n",
       "<td >\n",
       "-10.016316725971112\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Abortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds\n",
       "</td>\n",
       "<td >\n",
       "8\n",
       "</td>\n",
       "<td >\n",
       "5\n",
       "</td>\n",
       "<td >\n",
       "0.030090497737556562\n",
       "</td>\n",
       "<td >\n",
       "0.8381417989730835\n",
       "</td>\n",
       "<td >\n",
       "-9.163558569425538\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "10098\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Kamala Harris says abortion bans are creating &#39;a health care crisis&#39;\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "1\n",
       "</td>\n",
       "<td >\n",
       "0.01639344262295082\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "-10.678829270936067\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "11154\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Meet the anti-abortion group using white coats and research to advance its cause\n",
       "</td>\n",
       "<td >\n",
       "1\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.01639344262295082\n",
       "</td>\n",
       "<td >\n",
       "0.7592407464981079\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "3958\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Supreme Court signals it is likely to reject a challenge to abortion pill access\n",
       "</td>\n",
       "<td >\n",
       "2\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.016129032258064516\n",
       "</td>\n",
       "<td >\n",
       "0.8076152205467224\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "2292\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Ohio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "3\n",
       "</td>\n",
       "<td >\n",
       "0.015873015873015872\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "-9.7149595994016\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "6011\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Supreme Court wrestles with abortion clash over emergency room treatment for pregnant women\n",
       "</td>\n",
       "<td >\n",
       "3\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.015873015873015872\n",
       "</td>\n",
       "<td >\n",
       "0.812921941280365\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "452\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "64K women and girls became pregnant due to rape in states with abortion bans, study estimates\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "4\n",
       "</td>\n",
       "<td >\n",
       "0.015625\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "-9.163558569425538\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "3933\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "The Supreme Court puts the GOP’s abortion dilemma back in the spotlight: From the Politics Desk\n",
       "</td>\n",
       "<td >\n",
       "4\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.015625\n",
       "</td>\n",
       "<td >\n",
       "0.8234137296676636\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "7381\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Where abortion rights could be on the ballot this fall: From the Politics Desk\n",
       "</td>\n",
       "<td >\n",
       "5\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.015384615384615385\n",
       "</td>\n",
       "<td >\n",
       "0.8250945210456848\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "6680\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "More young people choosing permanent sterilization after abortion restrictions\n",
       "</td>\n",
       "<td >\n",
       "6\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.015151515151515152\n",
       "</td>\n",
       "<td >\n",
       "0.832757294178009\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "2646\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Trump campaign scrambles over abortion ban report as Democrats seize the moment\n",
       "</td>\n",
       "<td >\n",
       "9\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.014492753623188406\n",
       "</td>\n",
       "<td >\n",
       "0.8460941910743713\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9447\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "At a clinic in Hialeah, immigrants wrestle with Florida&#39;s new abortion ban\n",
       "</td>\n",
       "<td >\n",
       "10\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "<td >\n",
       "0.014285714285714285\n",
       "</td>\n",
       "<td >\n",
       "0.8477099537849426\n",
       "</td>\n",
       "<td >\n",
       "\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "<div style=\"text-align: right;\">\n",
       "13 rows × 7 columns\n",
       "</div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "\u001b[0m┌\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m──────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m──────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m──────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┐\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mid\u001b[0m   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mheadline\u001b[0m                                                                                              \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mvec_rank\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mfts_rank\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mcombined_rank\u001b[0m       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mvec_distance\u001b[0m      \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mfts_score\u001b[0m          \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m├\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m──────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m──────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m──────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┤\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mStates with abortion bans saw birth control prescriptions fall post-Dobbs, study finds                \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       7\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       2\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.031054405392392875\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.837793231010437\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-10.016316725971112\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAbortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       8\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       5\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.030090497737556562\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.8381417989730835\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m -9.163558569425538\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m10098\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mKamala Harris says abortion bans are creating 'a health care crisis'                                  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       1\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.01639344262295082\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-10.678829270936067\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m11154\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mMeet the anti-abortion group using white coats and research to advance its cause                      \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       1\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.01639344262295082\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.7592407464981079\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 3958\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mSupreme Court signals it is likely to reject a challenge to abortion pill access                      \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       2\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.016129032258064516\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.8076152205467224\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 2292\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mOhio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       3\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.015873015873015872\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m   -9.7149595994016\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 6011\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mSupreme Court wrestles with abortion clash over emergency room treatment for pregnant women           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       3\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.015873015873015872\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.812921941280365\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  452\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m64K women and girls became pregnant due to rape in states with abortion bans, study estimates         \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       4\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m            0.015625\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m -9.163558569425538\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 3933\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mThe Supreme Court puts the GOP’s abortion dilemma back in the spotlight: From the Politics Desk       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       4\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m            0.015625\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.8234137296676636\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 7381\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk                        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       5\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.015384615384615385\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.8250945210456848\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 6680\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mMore young people choosing permanent sterilization after abortion restrictions                        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       6\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.015151515151515152\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 0.832757294178009\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 2646\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mTrump campaign scrambles over abortion ban report as Democrats seize the moment                       \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m       9\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.014492753623188406\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.8460941910743713\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9447\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAt a clinic in Hialeah, immigrants wrestle with Florida's new abortion ban                            \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m      10\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m        \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.014285714285714285\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m0.8477099537849426\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m                   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m└\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m──────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m──────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m──────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┘\n",
       "\u001b[0m\u001b[0m"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".param set query abortion bans\n",
    "\n",
    "\n",
    ".param set weight_fts 1.0\n",
    ".param set weight_vec 1.0\n",
    ".param set rrf_k 60\n",
    ".param set k 10\n",
    "\n",
    "\n",
    "with vec_matches as (\n",
    "  select\n",
    "    article_id,\n",
    "    row_number() over (order by distance) as rank_number,\n",
    "    distance\n",
    "  from vec_headlines\n",
    "  where\n",
    "    headline_embedding match lembed(:query)\n",
    "    and k = :k\n",
    "  order by distance\n",
    "),\n",
    "fts_matches as (\n",
    "  select\n",
    "    rowid,\n",
    "    row_number() over (order by rank) as rank_number,\n",
    "    rank as score\n",
    "  from fts_headlines\n",
    "  where headline match :query\n",
    "  limit :k\n",
    "),\n",
    "final as (\n",
    "  select\n",
    "    articles.id,\n",
    "    articles.headline,\n",
    "    vec_matches.rank_number as vec_rank,\n",
    "    fts_matches.rank_number as fts_rank,\n",
    "    coalesce(1.0 / (:rrf_k + fts_matches.rank_number), 0.0) * :weight_fts\n",
    "    + coalesce(1.0 / (:rrf_k + vec_matches.rank_number), 0.0) * :weight_vec\n",
    "      as combined_rank,\n",
    "    vec_matches.distance as vec_distance,\n",
    "    fts_matches.score as fts_score\n",
    "  from fts_matches\n",
    "  full outer join vec_matches on vec_matches.article_id = fts_matches.rowid\n",
    "  join articles on articles.rowid = coalesce(fts_matches.rowid, vec_matches.article_id)\n",
    "  order by combined_rank desc\n",
    ")\n",
    "select * from final;\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Re-rank by semantics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table>\n",
       "<thead>\n",
       "<tr style=\"text-align: center;\">\n",
       "<th>\n",
       "id\n",
       "</th>\n",
       "<th>\n",
       "headline\n",
       "</th>\n",
       "<th>\n",
       "rowid\n",
       "</th>\n",
       "<th>\n",
       "fts_rank_number\n",
       "</th>\n",
       "<th>\n",
       "score\n",
       "</th>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td >\n",
       "452\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "64K women and girls became pregnant due to rape in states with abortion bans, study estimates\n",
       "</td>\n",
       "<td >\n",
       "452\n",
       "</td>\n",
       "<td >\n",
       "4\n",
       "</td>\n",
       "<td >\n",
       "-9.163558569425538\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "States with abortion bans saw birth control prescriptions fall post-Dobbs, study finds\n",
       "</td>\n",
       "<td >\n",
       "9776\n",
       "</td>\n",
       "<td >\n",
       "2\n",
       "</td>\n",
       "<td >\n",
       "-10.016316725971112\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "2292\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Ohio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\n",
       "</td>\n",
       "<td >\n",
       "2292\n",
       "</td>\n",
       "<td >\n",
       "3\n",
       "</td>\n",
       "<td >\n",
       "-9.7149595994016\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "10098\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Kamala Harris says abortion bans are creating &#39;a health care crisis&#39;\n",
       "</td>\n",
       "<td >\n",
       "10098\n",
       "</td>\n",
       "<td >\n",
       "1\n",
       "</td>\n",
       "<td >\n",
       "-10.678829270936067\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td style=\"text-align: left;\">\n",
       "Abortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds\n",
       "</td>\n",
       "<td >\n",
       "9187\n",
       "</td>\n",
       "<td >\n",
       "5\n",
       "</td>\n",
       "<td >\n",
       "-9.163558569425538\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "<div style=\"text-align: right;\">\n",
       "5 rows × 5 columns\n",
       "</div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "\u001b[0m┌\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────────\u001b[0m\u001b[0m┬\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┐\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mid\u001b[0m   \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mheadline\u001b[0m                                                                                              \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mrowid\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mfts_rank_number\u001b[0m\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m\u001b[0m\u001b[1mscore\u001b[0m              \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m├\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────────\u001b[0m\u001b[0m┼\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┤\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  452\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m64K women and girls became pregnant due to rape in states with abortion bans, study estimates         \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m  452\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m              4\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m -9.163558569425538\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mStates with abortion bans saw birth control prescriptions fall post-Dobbs, study finds                \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9776\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m              2\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-10.016316725971112\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 2292\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mOhio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 2292\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m              3\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m   -9.7149595994016\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m10098\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mKamala Harris says abortion bans are creating 'a health care crisis'                                  \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m10098\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m              1\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m-10.678829270936067\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0mAbortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds           \u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m 9187\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m              5\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m \u001b[0m\u001b[0m\u001b[0m -9.163558569425538\u001b[0m \u001b[0m\u001b[0m│\u001b[0m\u001b[0m\n",
       "\u001b[0m\u001b[0m\u001b[0m└\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m───────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────────\u001b[0m\u001b[0m┴\u001b[0m\u001b[0m─────────────────────\u001b[0m\u001b[0m┘\n",
       "\u001b[0m\u001b[0m"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ".param set query abortion bans\n",
    ".param set k 10\n",
    "\n",
    "\n",
    "with fts_matches as (\n",
    "  select\n",
    "    rowid,\n",
    "    row_number() over (order by rank) as fts_rank_number,\n",
    "    rank as score\n",
    "  from fts_headlines\n",
    "  where headline match :query\n",
    "  limit :k\n",
    "),\n",
    "final as (\n",
    "  select\n",
    "    articles.id,\n",
    "    articles.headline,\n",
    "    fts_matches.*\n",
    "  from fts_matches\n",
    "  left join articles on articles.rowid = fts_matches.rowid\n",
    "  order by vec_distance_cosine(lembed(:query), lembed(articles.headline))\n",
    ")\n",
    "select * from final;\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Solite",
   "language": "sql",
   "name": "solite"
  },
  "language_info": {
   "file_extension": ".sql",
   "mimetype": "text/x.sqlite",
   "name": "sql",
   "nb_converter": "script",
   "pygments_lexer": "sql",
   "version": "TODO"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}