import os
import time
import warnings

warnings.filterwarnings("ignore")
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import numpy as np
import polars as pl
import matplotlib.pyplot as plt

import common
import talentclef as tc

RESULTS = common.RESULTS_DIR
os.makedirs(RESULTS, exist_ok=True)
np.random.seed(0)

print("TalentCLEF data dir:", os.path.abspath(tc.TALENTCLEF_DIR))

TalentCLEF data dir: /home/ubuntu/claude_/multilingual-job-title-matching/data/raw/talentclef/TaskA

rows = []
for lang in tc.LANGUAGES:
    q, c, qr = tc.load_split(lang, "validation")
    rel = tc.qrels_to_dict(qr)
    n_rel = [len(v) for v in rel.values()]
    rows.append({
        "language": lang,
        "queries": q.height,
        "corpus_titles": c.height,
        "qrel_rows": qr.height,
        "queries_with_answers": len(rel),
        "median_correct_answers_per_query": float(np.median(n_rel)),
        "max_correct_answers_per_query": int(np.max(n_rel)),
    })
splits = pl.DataFrame(rows)
splits

q_en, c_en, qr_en = tc.load_split("english", "validation")
print("Example queries:")
print(q_en.head(5))
print("\nExample corpus titles:")
print(c_en.head(5))
print("\nThe answer key (qrels) — 'query 1 is correctly matched by corpus title 143':")
print(qr_en.head(5))

Example queries:
shape: (5, 2)
┌──────┬─────────────────────┐
│ q_id ┆ jobtitle            │
│ ---  ┆ ---                 │
│ i64  ┆ str                 │
╞══════╪═════════════════════╡
│ 1    ┆ nanny               │
│ 2    ┆ food technologist   │
│ 3    ┆ broadcast engineer  │
│ 4    ┆ automation engineer │
│ 5    ┆ veterinarian        │
└──────┴─────────────────────┘

Example corpus titles:
shape: (5, 2)
┌──────┬─────────────────────────────────┐
│ c_id ┆ jobtitle                        │
│ ---  ┆ ---                             │
│ i64  ┆ str                             │
╞══════╪═════════════════════════════════╡
│ 1    ┆ recording engineer              │
│ 2    ┆ director of taxation            │
│ 3    ┆ technical support representati… │
│ 4    ┆ hr manager                      │
│ 5    ┆ computer graphic artist         │
└──────┴─────────────────────────────────┘

The answer key (qrels) — 'query 1 is correctly matched by corpus title 143':
shape: (5, 3)
┌──────┬──────┬───────────┐
│ q_id ┆ c_id ┆ relevance │
│ ---  ┆ ---  ┆ ---       │
│ i64  ┆ i64  ┆ i64       │
╞══════╪══════╪═══════════╡
│ 1    ┆ 143  ┆ 1         │
│ 1    ┆ 150  ┆ 1         │
│ 1    ┆ 764  ┆ 1         │
│ 1    ┆ 870  ┆ 1         │
│ 1    ┆ 1464 ┆ 1         │
└──────┴──────┴───────────┘

# What does one query actually look like, end to end?
rel_en = tc.qrels_to_dict(qr_en)
c_lookup = dict(zip(c_en["c_id"].to_list(), c_en["jobtitle"].to_list()))
demo_q = q_en.row(0, named=True)
print(f"Query {demo_q['q_id']}: {demo_q['jobtitle']!r}")
print(f"Has {len(rel_en[demo_q['q_id']])} correct answers in the corpus. First 10:")
for cid in sorted(rel_en[demo_q["q_id"]])[:10]:
    print("   -", c_lookup[cid])

Query 1: 'nanny'
Has 17 correct answers in the corpus. First 10:
   - counselor
   - daycare teacher
   - assistant preschool teacher
   - babysitter
   - classroom assistant
   - childcare assistant
   - corps member
   - childcare provider
   - student placement coordinator
   - childcare worker

toy_ranked = ["A", "X", "B", "Y", "Z"]
toy_relevant = {"A", "B", "C", "D"}

ap = tc.average_precision(toy_ranked, toy_relevant)
rr = tc.reciprocal_rank(toy_ranked, toy_relevant)
p5 = tc.precision_at_k(toy_ranked, toy_relevant, 5)
expected_ap = (1 / 1 + 2 / 3) / 4

print(f"AP   = {ap:.4f}   (hand-computed: {expected_ap:.4f})   match: {abs(ap - expected_ap) < 1e-9}")
print(f"RR   = {rr:.4f}   (hand-computed: 1.0)                 match: {abs(rr - 1.0) < 1e-9}")
print(f"P@5  = {p5:.4f}   (hand-computed: 0.4)                 match: {abs(p5 - 0.4) < 1e-9}")

# A perfect ranking must score 1.0 on everything, and a ranking with no correct
# answers at all must score 0.0. If either of these fails the metric is broken.
perfect = tc.average_precision(["A", "B", "C", "D"], toy_relevant)
nothing = tc.average_precision(["X", "Y", "Z"], toy_relevant)
print(f"\nSanity: perfect ranking AP = {perfect:.4f} (must be 1.0)")
print(f"Sanity: zero-hit ranking AP = {nothing:.4f} (must be 0.0)")

AP   = 0.4167   (hand-computed: 0.4167)   match: True
RR   = 1.0000   (hand-computed: 1.0)                 match: True
P@5  = 0.4000   (hand-computed: 0.4)                 match: True

Sanity: perfect ranking AP = 1.0000 (must be 1.0)
Sanity: zero-hit ranking AP = 0.0000 (must be 0.0)

from rapidfuzz import fuzz, process


def run_random(queries, corpus, k=100, seed=0):
    rng = np.random.default_rng(seed)
    c_ids = corpus["c_id"].to_numpy()
    return {q: rng.permutation(c_ids)[:k].tolist() for q in queries["q_id"].to_list()}


def run_lexical(queries, corpus, k=100):
    """Rank the corpus by fuzzy string similarity to the query."""
    c_ids = corpus["c_id"].to_list()
    c_titles = corpus["jobtitle"].to_list()
    run = {}
    for q_id, q_title in zip(queries["q_id"].to_list(), queries["jobtitle"].to_list()):
        scored = process.extract(
            q_title, c_titles, scorer=fuzz.token_sort_ratio, limit=k
        )
        run[q_id] = [c_ids[idx] for _, _, idx in scored]
    return run


baseline_rows = []
for lang in tc.LANGUAGES:
    q, c, qr = tc.load_split(lang, "validation")
    rel = tc.qrels_to_dict(qr)
    for name, fn in [("random", run_random), ("lexical fuzzy", run_lexical)]:
        m = tc.evaluate_run(fn(q, c), rel)
        baseline_rows.append({"language": lang, "system": name, **{k: m[k] for k in ("n_queries", "MAP", "MRR", "P@5", "P@10")}})

baselines = pl.DataFrame(baseline_rows).sort(["language", "system"])
baselines

from sentence_transformers import SentenceTransformer

MODELS = [
    # (short name, HF id, query prefix, doc prefix, is_sentence_transformer)
    ("MiniLM (repo baseline)", "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "", "", True),
    ("JobBERT-v3", "TechWolf/JobBERT-v3", "", "", True),
    ("multilingual-e5-base", "intfloat/multilingual-e5-base", "query: ", "passage: ", True),
    ("JobBERT-v2 (English only)", "TechWolf/JobBERT-v2", "", "", True),
]

_model_cache = {}


def get_model(hf_id):
    if hf_id not in _model_cache:
        t0 = time.time()
        _model_cache[hf_id] = SentenceTransformer(hf_id, device="cpu")
        print(f"  loaded {hf_id} in {time.time() - t0:.1f}s")
    return _model_cache[hf_id]


def run_embedding(queries, corpus, hf_id, q_prefix="", d_prefix="", k=100):
    """Encode queries + corpus, rank corpus by cosine similarity. Returns
    (run, seconds_spent_encoding, corpus_embeddings, query_embeddings)."""
    model = get_model(hf_id)
    t0 = time.time()
    q_emb = common.encode_texts(model, [q_prefix + t for t in queries["jobtitle"].to_list()], show_progress_bar=False)
    c_emb = common.encode_texts(model, [d_prefix + t for t in corpus["jobtitle"].to_list()], show_progress_bar=False)
    encode_s = time.time() - t0

    c_ids = corpus["c_id"].to_numpy()
    top_idx, _ = common.batched_topk(q_emb, c_emb, k=min(k, c_emb.shape[0]))
    run = {q_id: c_ids[top_idx[i]].tolist() for i, q_id in enumerate(queries["q_id"].to_list())}
    return run, encode_s, c_emb, q_emb


emb_rows = []
for lang in tc.LANGUAGES:
    q, c, qr = tc.load_split(lang, "validation")
    rel = tc.qrels_to_dict(qr)
    print(f"\n--- {lang} ({q.height} queries, {c.height} corpus titles) ---")
    for name, hf_id, qp, dp, _ in MODELS:
        if "JobBERT-v2" in name and lang != "english":
            continue  # v2 is English-only; running it elsewhere would be a strawman
        run, enc_s, _, _ = run_embedding(q, c, hf_id, qp, dp)
        m = tc.evaluate_run(run, rel)
        emb_rows.append({"language": lang, "system": name,
                         **{k: m[k] for k in ("n_queries", "MAP", "MRR", "P@5", "P@10")},
                         "encode_seconds": round(enc_s, 1)})
        print(f"  {name:28s} MAP={m['MAP']:.4f}  MRR={m['MRR']:.4f}  ({enc_s:.1f}s)")

embeddings_df = pl.DataFrame(emb_rows)
embeddings_df

--- english (105 queries, 2619 corpus titles) ---

Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.

  loaded sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 in 6.8s

  MiniLM (repo baseline)       MAP=0.4803  MRR=0.7624  (8.6s)

  loaded TechWolf/JobBERT-v3 in 7.9s

  JobBERT-v3                   MAP=0.6161  MRR=0.8056  (22.6s)

  loaded intfloat/multilingual-e5-base in 7.5s

  multilingual-e5-base         MAP=0.4836  MRR=0.7985  (26.1s)

  loaded TechWolf/JobBERT-v2 in 5.0s

  JobBERT-v2 (English only)    MAP=0.6329  MRR=0.8302  (19.7s)

--- spanish (185 queries, 4661 corpus titles) ---

import torch
from transformers import AutoModel, AutoTokenizer


def mean_pooled_embeddings(hf_id, texts, batch_size=32, max_length=32):
    """Mean-pool a raw masked-LM's token embeddings into one vector per text,
    masking out padding so padded positions don't drag the average around."""
    tok = AutoTokenizer.from_pretrained(hf_id)
    mdl = AutoModel.from_pretrained(hf_id).eval()
    out = []
    with torch.no_grad():
        for i in range(0, len(texts), batch_size):
            batch = tok(texts[i:i + batch_size], padding=True, truncation=True,
                        max_length=max_length, return_tensors="pt")
            hidden = mdl(**batch).last_hidden_state
            mask = batch["attention_mask"].unsqueeze(-1).float()
            pooled = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1e-9)
            out.append(torch.nn.functional.normalize(pooled, p=2, dim=1).numpy())
    return np.vstack(out).astype(np.float32)


q, c, qr = tc.load_split("english", "validation")
rel = tc.qrels_to_dict(qr)
t0 = time.time()
q_emb = mean_pooled_embeddings("jhu-clsp/mmBERT-base", q["jobtitle"].to_list())
c_emb = mean_pooled_embeddings("jhu-clsp/mmBERT-base", c["jobtitle"].to_list())
mm_s = time.time() - t0

c_ids = c["c_id"].to_numpy()
top_idx, _ = common.batched_topk(q_emb, c_emb, k=100)
mm_run = {q_id: c_ids[top_idx[i]].tolist() for i, q_id in enumerate(q["q_id"].to_list())}
mm_metrics = tc.evaluate_run(mm_run, rel)

lex_en = tc.evaluate_run(run_lexical(q, c), rel)
best_en = embeddings_df.filter(pl.col("language") == "english")["MAP"].max()
print(f"mmBERT-base, mean-pooled, English : MAP = {mm_metrics['MAP']:.4f}  ({mm_s:.1f}s)")
print(f"  vs lexical fuzzy baseline       : MAP = {lex_en['MAP']:.4f}")
print(f"  vs best sentence-embedding model: MAP = {best_en:.4f}")
print("\nThis is the point: a strong multilingual encoder used the wrong way is not")
print("automatically a strong retriever. Architecture alone is not the deciding factor;")
print("what the model was *trained to do* is. mmBERT is an excellent starting point to")
print("fine-tune from (notebook 06 does exactly that) — but it is not a drop-in matcher.")

emb_rows.append({"language": "english", "system": "mmBERT-base (mean-pooled, no fine-tuning)",
                 **{k: mm_metrics[k] for k in ("n_queries", "MAP", "MRR", "P@5", "P@10")},
                 "encode_seconds": round(mm_s, 1)})

[transformers] ModernBertModel LOAD REPORT from: jhu-clsp/mmBERT-base
Key               | Status     |  | 
------------------+------------+--+-
decoder.bias      | UNEXPECTED |  | 
head.dense.weight | UNEXPECTED |  | 
head.norm.weight  | UNEXPECTED |  | 
decoder.weight    | UNEXPECTED |  | 

Notes:
- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.

[transformers] ModernBertModel LOAD REPORT from: jhu-clsp/mmBERT-base
Key               | Status     |  | 
------------------+------------+--+-
decoder.bias      | UNEXPECTED |  | 
head.dense.weight | UNEXPECTED |  | 
head.norm.weight  | UNEXPECTED |  | 
decoder.weight    | UNEXPECTED |  | 

Notes:
- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.

mmBERT-base, mean-pooled, English : MAP = 0.2873  (54.5s)
  vs lexical fuzzy baseline       : MAP = 0.1957
  vs best sentence-embedding model: MAP = 0.6329

This is the point: a strong multilingual encoder used the wrong way is not
automatically a strong retriever. Architecture alone is not the deciding factor;
what the model was *trained to do* is. mmBERT is an excellent starting point to
fine-tune from (notebook 06 does exactly that) — but it is not a drop-in matcher.

import faiss
import hnswlib

cand = pl.read_csv(os.path.join(common.DATA_DIR, "esco_en_candidates.csv"))
cand_emb = np.load(os.path.join(common.DATA_DIR, "esco_en_candidate_embeddings.npy")).astype(np.float32)
print(f"Scale-test corpus: {cand_emb.shape[0]:,} ESCO English titles, {cand_emb.shape[1]} dimensions")

model = get_model("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
probe_titles = q["jobtitle"].to_list()
probe = common.encode_texts(model, probe_titles, show_progress_bar=False)
K = 10

# --- Exact reference (FAISS flat inner product; vectors are L2-normalised so
# inner product == cosine similarity) ---
t0 = time.time(); index_flat = faiss.IndexFlatIP(cand_emb.shape[1]); index_flat.add(cand_emb); build_flat = time.time() - t0
t0 = time.time(); _, exact_idx = index_flat.search(probe, K); q_flat = time.time() - t0

ann_rows = [{"index": "FAISS IndexFlatIP (exact)", "build_s": round(build_flat, 3),
             "query_ms_per_query": round(q_flat / len(probe) * 1000, 3), "recall@10_vs_exact": 1.0}]

# --- FAISS HNSW, at several search-effort settings ---
t0 = time.time()
index_hnsw = faiss.IndexHNSWFlat(cand_emb.shape[1], 32, faiss.METRIC_INNER_PRODUCT)
index_hnsw.hnsw.efConstruction = 200
index_hnsw.add(cand_emb)
build_hnsw = time.time() - t0

for ef in (16, 32, 64, 128):
    index_hnsw.hnsw.efSearch = ef
    t0 = time.time(); _, approx_idx = index_hnsw.search(probe, K); qt = time.time() - t0
    recall = np.mean([len(set(approx_idx[i]) & set(exact_idx[i])) / K for i in range(len(probe))])
    ann_rows.append({"index": f"FAISS HNSW (efSearch={ef})", "build_s": round(build_hnsw, 3),
                     "query_ms_per_query": round(qt / len(probe) * 1000, 3),
                     "recall@10_vs_exact": round(float(recall), 4)})

# --- hnswlib, the standalone implementation ---
t0 = time.time()
p = hnswlib.Index(space="ip", dim=cand_emb.shape[1])
p.init_index(max_elements=cand_emb.shape[0], ef_construction=200, M=32)
p.add_items(cand_emb, np.arange(cand_emb.shape[0]))
build_hl = time.time() - t0
for ef in (16, 64):
    p.set_ef(max(ef, K))
    t0 = time.time(); labels, _ = p.knn_query(probe, k=K); qt = time.time() - t0
    recall = np.mean([len(set(labels[i]) & set(exact_idx[i])) / K for i in range(len(probe))])
    ann_rows.append({"index": f"hnswlib (ef={ef})", "build_s": round(build_hl, 3),
                     "query_ms_per_query": round(qt / len(probe) * 1000, 3),
                     "recall@10_vs_exact": round(float(recall), 4)})

ann_df = pl.DataFrame(ann_rows)
ann_df

Scale-test corpus: 16,919 ESCO English titles, 384 dimensions

fig, ax = plt.subplots(1, 2, figsize=(13, 4.5))
sub = ann_df.filter(pl.col("index").str.contains("HNSW"))
ax[0].plot(sub["query_ms_per_query"], sub["recall@10_vs_exact"], "o-", color="#4C78A8")
for r in sub.iter_rows(named=True):
    ax[0].annotate(r["index"].split("=")[-1].rstrip(")"), (r["query_ms_per_query"], r["recall@10_vs_exact"]),
                   textcoords="offset points", xytext=(6, -10), fontsize=9)
exact_ms = ann_df.filter(pl.col("index").str.contains("exact"))["query_ms_per_query"][0]
ax[0].axvline(exact_ms, ls="--", color="#E45756", label=f"exact search ({exact_ms:.2f} ms)")
ax[0].set_xlabel("query time (ms per query)"); ax[0].set_ylabel("recall@10 vs exact")
ax[0].set_title(f"ANN accuracy/speed trade-off\n({cand_emb.shape[0]:,} ESCO titles, labels = efSearch)")
ax[0].legend(); ax[0].grid(alpha=.3)

en = embeddings_df.filter(pl.col("language") == "english").sort("MAP")
names = en["system"].to_list() + ["lexical fuzzy (baseline)", "random (floor)"]
vals = en["MAP"].to_list() + [
    baselines.filter((pl.col("language") == "english") & (pl.col("system") == "lexical fuzzy"))["MAP"][0],
    baselines.filter((pl.col("language") == "english") & (pl.col("system") == "random"))["MAP"][0],
]
order = np.argsort(vals)
ax[1].barh([names[i] for i in order], [vals[i] for i in order],
           color=["#B0B0B0" if "baseline" in names[i] or "floor" in names[i] else "#4C78A8" for i in order])
ax[1].set_xlabel("MAP (English validation split)")
ax[1].set_title(f"English retrieval quality\n({en['n_queries'][0]} queries, answer key by TalentCLEF organizers)")
ax[1].grid(alpha=.3, axis="x")
plt.tight_layout()
plt.savefig(os.path.join(RESULTS, "talentclef_retrieval.png"), dpi=110, bbox_inches="tight")
plt.show()

esco_occ, esco_titles = common.load_esco()
pref = esco_titles.filter(pl.col("label_type") == "preferred")

cross_rows = []
for tgt in ("fr", "de", "es", "it", "pl", "zh"):
    src_df = pref.filter(pl.col("language") == "en").group_by("uri").agg(pl.col("title").first())
    tgt_df = pref.filter(pl.col("language") == tgt).group_by("uri").agg(pl.col("title").first())
    joined = src_df.join(tgt_df, on="uri", how="inner", suffix="_tgt")
    if joined.height < 50:
        continue
    for name, hf_id, qp, dp, _ in MODELS:
        if "JobBERT-v2" in name:
            continue  # English-only by construction
        model = get_model(hf_id)
        qe = common.encode_texts(model, [qp + t for t in joined["title"].to_list()], show_progress_bar=False)
        ce = common.encode_texts(model, [dp + t for t in joined["title_tgt"].to_list()], show_progress_bar=False)
        top_idx, _ = common.batched_topk(qe, ce, k=5)
        truth = np.arange(joined.height)  # row i of the query side matches row i of the target side
        cross_rows.append({
            "target_language": tgt, "system": name, "n_pairs": joined.height,
            "accuracy@1": round(float((top_idx[:, 0] == truth).mean()), 4),
            "accuracy@5": round(float(np.mean([truth[i] in top_idx[i] for i in range(len(truth))])), 4),
        })
    print(f"  {tgt}: {joined.height} aligned ESCO occupation pairs")

cross_df = pl.DataFrame(cross_rows).sort(["target_language", "system"])
cross_df

  fr: 1699 aligned ESCO occupation pairs

  de: 1699 aligned ESCO occupation pairs

  es: 1699 aligned ESCO occupation pairs

  it: 1699 aligned ESCO occupation pairs

  pl: 1699 aligned ESCO occupation pairs

best_name, best_id, best_qp, best_dp, _ = MODELS[1]  # JobBERT-v3
q, c, qr = tc.load_split("english", "validation")
rel = tc.qrels_to_dict(qr)
run, _, _, _ = run_embedding(q, c, best_id, best_qp, best_dp)
q_lookup = dict(zip(q["q_id"].to_list(), q["jobtitle"].to_list()))

misses = [(qid, tc.average_precision(ranked, rel.get(qid, set())))
          for qid, ranked in run.items() if rel.get(qid)]
misses.sort(key=lambda x: x[1])

print(f"Worst-scoring queries for {best_name} (English validation):\n")
for qid, ap in misses[:6]:
    top1 = run[qid][0]
    correct = sorted(rel[qid])[:3]
    print(f"  query: {q_lookup[qid]!r}   (AP={ap:.3f})")
    print(f"    ranked #1: {c_lookup[top1]!r}  {'CORRECT' if top1 in rel[qid] else 'WRONG'}")
    print(f"    should have found e.g.: {[c_lookup[x] for x in correct]}")
    print()

Worst-scoring queries for JobBERT-v3 (English validation):

  query: 'biomedical engineer'   (AP=0.000)
    ranked #1: 'medical engineer'  WRONG
    should have found e.g.: ['equipment maintenance technician']

  query: 'automotive engineer'   (AP=0.073)
    ranked #1: 'propulsion engineer'  WRONG
    should have found e.g.: ['research and development engineer', 'design release engineer', 'design and development engineer']

  query: 'wireless engineer'   (AP=0.185)
    ranked #1: 'wireless engineer'  WRONG
    should have found e.g.: ['wireless specialist', 'wireless product manager', 'network performance engineer']

  query: 'chemical operator'   (AP=0.253)
    ranked #1: 'technical operator'  WRONG
    should have found e.g.: ['chemical engineer', 'plant process engineer', 'process specialist']

  query: 'film director'   (AP=0.270)
    ranked #1: 'film director'  WRONG
    should have found e.g.: ['location scout', 'line producer', 'assistant producer']

  query: 'cashier'   (AP=0.277)
    ranked #1: 'cashier'  WRONG
    should have found e.g.: ['cashier customer service', 'assistant frontend manager', 'customer service representative']

final = pl.concat([
    baselines.with_columns(pl.lit(None).cast(pl.Float64).alias("encode_seconds")),
    pl.DataFrame(emb_rows),
], how="diagonal").sort(["language", "MAP"], descending=[False, True])

final.write_csv(os.path.join(RESULTS, "talentclef_retrieval_results.csv"))
ann_df.write_csv(os.path.join(RESULTS, "ann_index_tradeoff.csv"))
cross_df.write_csv(os.path.join(RESULTS, "cross_lingual_model_comparison.csv"))
tc.write_trec_run(run, os.path.join(RESULTS, "talentclef_english_jobbert_v3.trec"), tag="jobbert-v3")

print("Wrote: talentclef_retrieval_results.csv, ann_index_tradeoff.csv,")
print("       cross_lingual_model_comparison.csv, talentclef_english_jobbert_v3.trec")
final

Wrote: talentclef_retrieval_results.csv, ann_index_tradeoff.csv,
       cross_lingual_model_comparison.csv, talentclef_english_jobbert_v3.trec

Term	What it actually means
Embedding	A list of numbers (here, 384 or 768 of them) representing a title's meaning. Similar meanings → similar numbers.
Cosine similarity	A number from -1 to 1 measuring how close two embeddings point. 1 = same direction = same meaning.
Retrieval	Given one query title, rank all corpus titles by similarity and return the best ones.
Query / corpus	The title you search with / the pile of titles you search among.
qrels	"Query relevance judgments" — the human-made answer key saying which corpus titles are genuinely correct for each query.
Brute force / exact search	Compare the query against every single corpus item. Always correct, gets slow as the corpus grows.
ANN (approximate nearest neighbour)	A pre-built index that finds almost the same answers much faster by not comparing against everything. FAISS and HNSW are two such indexes.

language	system	n_queries	MAP	MRR	P@5	P@10
str	str	i64	f64	f64	f64	f64
"chinese"	"lexical fuzzy"	103	0.288586	0.726489	0.520388	0.421359
"chinese"	"random"	103	0.002501	0.03461	0.007767	0.005825
"english"	"lexical fuzzy"	105	0.195719	0.61522	0.392381	0.321905
"english"	"random"	105	0.001915	0.033479	0.009524	0.012381
"german"	"lexical fuzzy"	203	0.159678	0.460214	0.403941	0.386207
"german"	"random"	203	0.000898	0.02842	0.008867	0.008867
"spanish"	"lexical fuzzy"	185	0.15883	0.539098	0.443243	0.371892
"spanish"	"random"	185	0.001169	0.041788	0.010811	0.010811

Model	What it is	Why included
`paraphrase-multilingual-MiniLM-L12-v2`	General multilingual sentence embeddings, small (384-dim)	The baseline this repo used in notebooks 03/04 — the thing to beat
`TechWolf/JobBERT-v3`	Trained specifically on 21M+ job titles, contrastively, for en/de/es/zh	Reported state of the art on this exact benchmark
`intfloat/multilingual-e5-base`	Strong general-purpose multilingual retrieval model	Fair "good general model" comparison — is domain training actually needed?
`TechWolf/JobBERT-v2`	English-only predecessor of v3	Shows what the multilingual upgrade bought
`jhu-clsp/mmBERT-base`	A raw masked language model, not a sentence-embedding model	Deliberate demonstration — see section 5

language	system	n_queries	MAP	MRR	P@5	P@10	encode_seconds
str	str	i64	f64	f64	f64	f64	f64
"english"	"MiniLM (repo baseline)"	105	0.480345	0.762391	0.653333	0.601905	8.6
"english"	"JobBERT-v3"	105	0.616058	0.805556	0.742857	0.667619	22.6
"english"	"multilingual-e5-base"	105	0.483613	0.798503	0.655238	0.57619	26.1
"english"	"JobBERT-v2 (English only)"	105	0.632922	0.830159	0.733333	0.673333	19.7
"spanish"	"MiniLM (repo baseline)"	185	0.337452	0.54464	0.607568	0.572973	13.6
…	…	…	…	…	…	…	…
"german"	"JobBERT-v3"	203	0.414462	0.506489	0.567488	0.598522	51.4
"german"	"multilingual-e5-base"	203	0.254343	0.517055	0.520197	0.495567	56.9
"chinese"	"MiniLM (repo baseline)"	103	0.415929	0.771904	0.590291	0.512621	6.8
"chinese"	"JobBERT-v3"	103	0.569801	0.803463	0.718447	0.633981	21.9
"chinese"	"multilingual-e5-base"	103	0.43368	0.781877	0.621359	0.532039	25.9

index	build_s	query_ms_per_query	recall@10_vs_exact
str	f64	f64	f64
"FAISS IndexFlatIP (exact)"	0.016	1.242	1.0
"FAISS HNSW (efSearch=16)"	2.111	0.03	0.9905
"FAISS HNSW (efSearch=32)"	2.111	0.042	0.9971
"FAISS HNSW (efSearch=64)"	2.111	0.048	1.0
"FAISS HNSW (efSearch=128)"	2.111	0.09	1.0
"hnswlib (ef=16)"	1.818	0.02	0.979
"hnswlib (ef=64)"	1.818	0.055	1.0

05 — Retrieval with embeddings and ANN indexes, scored on TalentCLEF 2025¶

The vocabulary, in plain English¶

What you can do with this notebook¶

1. The data¶

2. The metrics — proved on a toy example, not taken on faith¶

3. Baselines first — so every later number means something¶

4. Embedding retrieval¶

5. Why `mmBERT-base` is not in that table — a deliberate lesson¶

6. ANN indexes — FAISS and HNSW¶

7. Cross-lingual retrieval — query in one language, corpus in another¶

8. Error analysis — what the best model actually gets wrong¶

9. Save results¶

10. What this notebook established¶

language	queries	corpus_titles	qrel_rows	queries_with_answers	median_correct_answers_per_query	max_correct_answers_per_query
str	i64	i64	i64	i64	f64	i64
"english"	105	2619	2420	105	21.0	53
"spanish"	185	4661	7579	185	34.0	138
"german"	203	4729	8417	203	35.0	120
"chinese"	103	2513	2319	103	20.0	73

target_language	system	n_pairs	accuracy@1	accuracy@5
str	str	i64	f64	f64
"de"	"JobBERT-v3"	1699	0.6004	0.794
"de"	"MiniLM (repo baseline)"	1699	0.3579	0.538
"de"	"multilingual-e5-base"	1699	0.4797	0.6675
"es"	"JobBERT-v3"	1699	0.6957	0.8452
"es"	"MiniLM (repo baseline)"	1699	0.5285	0.7263
…	…	…	…	…
"it"	"MiniLM (repo baseline)"	1699	0.4567	0.6345
"it"	"multilingual-e5-base"	1699	0.5009	0.6922
"pl"	"JobBERT-v3"	1699	0.5668	0.754
"pl"	"MiniLM (repo baseline)"	1699	0.4785	0.658
"pl"	"multilingual-e5-base"	1699	0.4562	0.651

05 — Retrieval with embeddings and ANN indexes, scored on TalentCLEF 2025¶

The vocabulary, in plain English¶

What you can do with this notebook¶

1. The data¶

2. The metrics — proved on a toy example, not taken on faith¶

3. Baselines first — so every later number means something¶

4. Embedding retrieval¶

5. Why mmBERT-base is not in that table — a deliberate lesson¶

6. ANN indexes — FAISS and HNSW¶

7. Cross-lingual retrieval — query in one language, corpus in another¶

8. Error analysis — what the best model actually gets wrong¶

9. Save results¶

10. What this notebook established¶

5. Why `mmBERT-base` is not in that table — a deliberate lesson¶