Modernize: latest bertopic/streamlit, py3.12, cache_resource with full settings key, drop altair<5 pin

README.md

@@ -4,6 +4,7 @@ emoji: 🅱️ 🇪 🇷 🇹
 colorFrom: gray
 colorTo: gray
 sdk: streamlit
+python_version: "3.12"
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -1,6 +1,5 @@
 from bertopic import BERTopic
 import streamlit as st
-import streamlit.components.v1 as components
 from datasets import load_dataset
 import pandas as pd
 from sentence_transformers import SentenceTransformer
@@ -93,27 +92,29 @@ form.form_submit_button("Submit")
 
 
 
-@st.cache
+@st.cache_data
 def load_and_process_data(path, name, streaming, split_name, number_of_records):
-    dataset = load_dataset(path = path, name = name, streaming=streaming)
+    dataset = load_dataset(path = path, name = name or None, streaming=streaming)
     #return list(dataset)
     dataset_head = dataset[split_name].take(number_of_records)
     df = pd.DataFrame.from_dict(dataset_head)
     return df
 
 
+## The sub-models are built inside the cached function so that every setting forms part
+## of the cache key - st.cache_resource hashes arguments only, unlike legacy st.cache
+@st.cache_resource
+def load_model(model_name, use_topic_reduction, number_of_topics, use_random_seed,
+               hdbscan_min_cluster_size, hdbscan_min_samples, hdbscan_metric,
+               umap_n_neighbors, umap_n_components, umap_min_dist, umap_metric,
+               cv_lowercase, cv_ngram_min, cv_ngram_max, cv_analyzer, cv_max_df, cv_min_df):
+    hdbscan_model = HDBSCAN(min_cluster_size=hdbscan_min_cluster_size, min_samples = hdbscan_min_samples, metric=hdbscan_metric, prediction_data=True)
+    if use_random_seed:
+        umap_model = UMAP(n_neighbors=umap_n_neighbors, n_components=umap_n_components, min_dist=umap_min_dist, metric=umap_metric, random_state = 42)
+    else:
+        umap_model = UMAP(n_neighbors=umap_n_neighbors, n_components=umap_n_components, min_dist=umap_min_dist, metric=umap_metric)
+    vectorizer_model = CountVectorizer(lowercase = cv_lowercase, ngram_range=(cv_ngram_min, cv_ngram_max), analyzer=cv_analyzer, max_df=cv_max_df, min_df=cv_min_df, stop_words="english")
 
-hdbscan_model = HDBSCAN(min_cluster_size=hdbscan_min_cluster_size, min_samples = hdbscan_min_samples, metric=hdbscan_metric, prediction_data=True)
-if use_random_seed:
-    umap_model = UMAP(n_neighbors=umap_n_neighbors, n_components=umap_n_components, min_dist=umap_min_dist, metric=umap_metric, random_state = 42)
-else:
-    umap_model = UMAP(n_neighbors=umap_n_neighbors, n_components=umap_n_components, min_dist=umap_min_dist, metric=umap_metric)
-vectorizer_model = CountVectorizer(lowercase = cv_lowercase, ngram_range=(cv_ngram_min, cv_ngram_max), analyzer=cv_analyzer, max_df=cv_max_df, min_df=cv_min_df, stop_words="english")
-
-
-
-@st.cache(allow_output_mutation=True)
-def load_model(model_name, hdbscan_model=hdbscan_model, umap_model=umap_model, vectorizer_model=vectorizer_model, use_topic_reduction = use_topic_reduction, number_of_topics = number_of_topics):
     sentence_model = SentenceTransformer(model_name)
     if use_topic_reduction == "Auto":
         kw_model = BERTopic(embedding_model=sentence_model, umap_model = umap_model, hdbscan_model = hdbscan_model, vectorizer_model = vectorizer_model, nr_topics = "auto", calculate_probabilities = True)
@@ -123,13 +124,19 @@ def load_model(model_name, hdbscan_model=hdbscan_model, umap_model=umap_model, v
         kw_model = BERTopic(embedding_model=sentence_model, umap_model = umap_model, hdbscan_model = hdbscan_model, vectorizer_model = vectorizer_model, calculate_probabilities = True)
     return kw_model
 
-@st.cache()
-def fit_transform(model, docs):
-    topics, probs = model.fit_transform(docs)
+## _model is excluded from the cache key by the underscore; cache_key carries the settings
+@st.cache_data(show_spinner="Fitting BERTopic...")
+def fit_transform(_model, docs, cache_key):
+    topics, probs = _model.fit_transform(docs)
     return topics, probs
 
 
-model = load_model(model_name=model_name)
+model_settings = (model_name, use_topic_reduction, number_of_topics, use_random_seed,
+                  hdbscan_min_cluster_size, hdbscan_min_samples, hdbscan_metric,
+                  umap_n_neighbors, umap_n_components, umap_min_dist, umap_metric,
+                  cv_lowercase, cv_ngram_min, cv_ngram_max, cv_analyzer, cv_max_df, cv_min_df)
+
+model = load_model(*model_settings)
 
 df = load_and_process_data(dataset_name, dataset_name_2, True, split_name, number_of_records)
 
@@ -139,7 +146,7 @@ st.header("Original Dataset")
 st.write(df)
 
 
-topics, probs = fit_transform(model, X)
+topics, probs = fit_transform(model, X, model_settings)
 
 st.header("Topic assignment for each example")
 st.write(topics)

requirements.txt

@@ -1,7 +1,10 @@
 streamlit
-pandas
 bertopic
+sentence-transformers
+scikit-learn
+umap-learn
+hdbscan
 datasets
+pandas
 numpy
-numba
-altair<5
+plotly