scripts/verify_citations.py
#!/usr/bin/env python3
"""Verify that references exist and match their metadata (Crossref + OpenAlex).
Catches hallucinated or corrupted citations: DOIs that do not resolve, DOIs that
point to a different paper, wrong years/first authors, and titles with no match.
Usage:
python verify_citations.py refs.bib # verify every entry
python verify_citations.py refs.bib --keys a2020,b2021 # subset
python verify_citations.py --doi 10.1038/nature14539 # one or more DOIs
python verify_citations.py refs.bib --json report.json
Set CROSSREF_MAILTO=you@example.org to use Crossref's polite pool (faster, fewer 429s).
Exit codes: 0 all verified, 1 problems found, 2 bad input. Standard library only.
"""
from __future__ import annotations
import argparse
import difflib
import json
import os
import re
import sys
import time
import unicodedata
import urllib.error
import urllib.parse
import urllib.request
for _stream in (sys.stdout, sys.stderr):
if hasattr(_stream, "reconfigure"):
_stream.reconfigure(encoding="utf-8", errors="replace")
UA = "research-agent-skills-citation-verification/1.0 (https://github.com/KalarisLabs/research-agent-skills)"
TITLE_OK, TITLE_WEAK = 0.90, 0.75
def http_json(url: str, retries: int = 3) -> dict | None:
"""Handle http json."""
mailto = os.environ.get("CROSSREF_MAILTO")
headers = {"User-Agent": UA + (f" mailto:{mailto}" if mailto else ""), "Accept": "application/json"}
for attempt in range(retries):
try:
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=20) as r: # noqa: S310
return json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
if exc.code == 404:
return None
if exc.code in (429, 500, 502, 503, 504) and attempt < retries - 1:
time.sleep(2 ** attempt * 2)
continue
raise
except urllib.error.URLError:
if attempt < retries - 1:
time.sleep(2 ** attempt)
continue
raise
return None
# --- minimal BibTeX reader (same grammar as bibtex-hygiene/scripts/bib_lint.py) ---
def _value(text: str, i: int) -> tuple[str, int]:
"""Handle value."""
parts = []
while True:
while i < len(text) and text[i].isspace():
i += 1
if text[i] == "{":
depth, j = 0, i
while j < len(text):
if text[j] == "{":
depth += 1
elif text[j] == "}":
depth -= 1
if depth == 0:
break
j += 1
parts.append(text[i + 1:j])
i = j + 1
elif text[i] == '"':
j = text.index('"', i + 1)
parts.append(text[i + 1:j])
i = j + 1
else:
m = re.match(r"[^,}\s#]+", text[i:])
parts.append(m.group(0) if m else "")
i += m.end() if m else 1
while i < len(text) and text[i].isspace():
i += 1
if i < len(text) and text[i] == "#":
i += 1
continue
return "".join(parts), i
def parse_bib(text: str) -> list[dict]:
"""Parse bib."""
out = []
for m in re.finditer(r"@(\w+)\s*[{(]\s*([^,\s]+)\s*,", text):
if m.group(1).lower() in ("comment", "preamble", "string"):
continue
entry, i = {"_type": m.group(1).lower(), "_key": m.group(2)}, m.end()
while True:
fm = re.match(r"\s*([A-Za-z][\w:-]*)\s*=\s*", text[i:])
if not fm:
break
v, i = _value(text, i + fm.end())
entry[fm.group(1).lower()] = re.sub(r"\s+", " ", v).strip()
cm = re.match(r"\s*,", text[i:])
if not cm:
break
i += cm.end()
out.append(entry)
return out
def norm(s: str) -> str:
"""Handle norm."""
s = re.sub(r"\\[a-zA-Z]+\s*|[{}$\\]", "", s or "")
s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode().lower()
return re.sub(r"[^a-z0-9]+", " ", s).strip()
def similarity(a: str, b: str) -> float:
"""Handle similarity."""
return difflib.SequenceMatcher(None, norm(a), norm(b)).ratio()
def first_surname(authors: str) -> str:
"""Handle first surname."""
first = (authors or "").split(" and ")[0].strip()
if "," in first:
surname = first.split(",")[0]
elif first.split():
surname = first.split()[-1]
else:
surname = ""
return norm(surname)
def crossref_record(doi: str) -> dict | None:
"""Handle crossref record."""
data = http_json(f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='/')}")
return data.get("message") if data else None
def crossref_search(title: str, author: str) -> list[dict]:
"""Handle crossref search."""
q = urllib.parse.urlencode({"query.bibliographic": f"{title} {author}".strip(), "rows": 3,
"select": "DOI,title,author,issued,container-title,type"})
data = http_json(f"https://api.crossref.org/works?{q}")
return (data or {}).get("message", {}).get("items", [])
def openalex_search(title: str) -> list[dict]:
"""Handle openalex search."""
q = urllib.parse.urlencode({"search": title, "per-page": 3, "select": "id,doi,title,publication_year,authorships"})
data = http_json(f"https://api.openalex.org/works?{q}")
return (data or {}).get("results", [])
def arxiv_search(title: str, author: str) -> list[dict]:
"""arXiv Atom API; many ML/physics papers exist only as preprints or DOI-less proceedings."""
import xml.etree.ElementTree as ET # response from export.arxiv.org (trusted, no DTDs/entities used)
query = f'ti:"{" ".join(norm(title).split()[:15])}"' + (f" AND au:{author}" if author else "")
url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode({"search_query": query, "max_results": 5})
req = urllib.request.Request(url, headers={"User-Agent": UA})
with urllib.request.urlopen(req, timeout=20) as r: # noqa: S310
root = ET.fromstring(r.read()) # noqa: S314
ns = {"a": "http://www.w3.org/2005/Atom"}
out = []
for entry in root.findall("a:entry", ns):
arxiv_id = re.sub(r"v\d+$", "", (entry.findtext("a:id", "", ns) or "").rsplit("/abs/", 1)[-1])
first = entry.find("a:author/a:name", ns)
out.append({"doi": f"10.48550/arXiv.{arxiv_id}" if arxiv_id else "",
"title": " ".join((entry.findtext("a:title", "", ns) or "").split()),
"year": int((entry.findtext("a:published", "", ns) or "0")[:4] or 0) or None,
"first_author": norm(first.text.split()[-1]) if first is not None and first.text else ""})
return out
def cr_fields(rec: dict) -> dict:
"""Handle cr fields."""
authors = rec.get("author") or []
year = None
for k in ("published-print", "published-online", "issued", "created"):
parts = (rec.get(k) or {}).get("date-parts") or [[None]]
if parts[0] and parts[0][0]:
year = parts[0][0]
break
return {"doi": rec.get("DOI", ""), "title": (rec.get("title") or [""])[0], "year": year,
"first_author": norm(authors[0].get("family", "")) if authors else "",
"venue": (rec.get("container-title") or [""])[0]}
def _oa_first_author(item: dict) -> str:
"""Return the normalised surname of the first author in an OpenAlex work record."""
first = ((item.get("authorships") or [{}])[0].get("author") or {}).get("display_name", "")
return norm(first.split(" ")[-1]) if first else ""
def _lookup_doi(doi: str) -> dict | None:
"""Resolve a DOI via Crossref then OpenAlex and return normalised fields, or None if unresolved."""
rec = crossref_record(doi)
if rec is not None:
return cr_fields(rec)
oa = http_json(f"https://api.openalex.org/works/doi:{urllib.parse.quote(doi, safe='/')}")
if oa is None:
return None
return {"doi": doi, "title": oa.get("title") or "", "year": oa.get("publication_year"),
"first_author": _oa_first_author(oa)}
def _doi_problems(title: str, year: str, author: str, got: dict) -> list[str]:
"""Compare a bib entry with the record its DOI resolves to and list every disagreement."""
problems = []
if title:
sim = similarity(title, got["title"])
if sim < TITLE_WEAK:
problems.append(f"DOI points to a different work: '{got['title']}' (similarity {sim:.2f})")
elif sim < TITLE_OK:
problems.append(f"title differs from record: '{got['title']}' (similarity {sim:.2f})")
if year and got.get("year") and abs(int(year) - int(got["year"])) > 1:
problems.append(f"year {year} but record says {got['year']}")
if author and got.get("first_author") and author != got["first_author"]:
problems.append(f"first author '{author}' but record says '{got['first_author']}'")
return problems
def _check_by_doi(res: dict, title: str, doi: str, year: str, author: str) -> dict:
"""Fill the result for an entry that carries a DOI."""
got = _lookup_doi(doi)
if got is None:
res["status"] = "doi-not-found"
res["problems"].append(f"DOI {doi} does not resolve in Crossref or OpenAlex")
return res
res["problems"].extend(_doi_problems(title, year, author, got))
res["status"] = "verified" if not res["problems"] else "mismatch"
return res
def _title_candidates(title: str, author: str) -> list[dict]:
"""Collect candidate records for a DOI-less entry from Crossref and OpenAlex title searches."""
candidates = [cr_fields(item) for item in crossref_search(title, author)]
for item in openalex_search(title):
candidates.append({"doi": (item.get("doi") or "").replace("https://doi.org/", ""), "title": item.get("title") or "",
"year": item.get("publication_year"), "first_author": _oa_first_author(item)})
return candidates
def _agrees(year: str, author: str, c: dict) -> tuple[bool, bool]:
"""Return whether a candidate agrees with the entry on year (within one) and first author."""
year_ok = not year or not c.get("year") or abs(int(year) - int(c["year"])) <= 1
author_ok = not author or not c.get("first_author") or author == c["first_author"]
return year_ok, author_ok
def _find_best(title: str, year: str, author: str) -> dict | None:
"""Pick the best candidate by title similarity plus year and first-author agreement."""
# Popular titles are reused (reprints, talks, homonymous papers), so rank candidates by
# title similarity plus agreement on year and first author, not by title alone.
candidates = _title_candidates(title, author)
def score(c: dict) -> float:
"""Score a candidate by title similarity plus bonuses for year and author agreement."""
year_ok, author_ok = _agrees(year, author, c)
return similarity(title, c["title"]) + 0.2 * year_ok + 0.2 * author_ok
best = max(candidates, key=score, default=None)
if not best or similarity(title, best["title"]) < TITLE_OK or not all(_agrees(year, author, best)):
try:
candidates += arxiv_search(title, author)
except (urllib.error.URLError, TimeoutError, ValueError):
pass
best = max(candidates, key=score, default=None)
return best
def _classify_title_match(res: dict, best: dict, title: str, year: str, author: str) -> dict:
"""Classify a candidate whose title is at least as similar as TITLE_OK."""
year_ok, author_ok = _agrees(year, author, best)
res["suggestion"] = best
if year_ok and author_ok:
res["status"] = "found-add-doi" if best.get("doi") else "verified"
return res
res["status"] = "uncertain"
if not year_ok:
res["problems"].append(f"title matches but year {year} vs record {best['year']}")
if not author_ok:
res["problems"].append(f"title matches but first author '{author}' vs record '{best['first_author']}'")
return res
def _classify_match(res: dict, best: dict | None, title: str, year: str, author: str) -> dict:
"""Set status, suggestion and problems on the result from the best candidate found."""
best_sim = similarity(title, best["title"]) if best else 0.0
if best and best_sim >= TITLE_OK:
return _classify_title_match(res, best, title, year, author)
if best and best_sim >= TITLE_WEAK:
res["status"] = "uncertain"
res["suggestion"] = best
res["problems"].append(f"closest match '{best['title']}' (similarity {best_sim:.2f}); confirm manually")
return res
res["status"] = "not-found"
res["problems"].append("no matching record in Crossref or OpenAlex (possible fabricated or grey-literature reference)")
return res
def check_entry(e: dict) -> dict:
"""Check one bib entry against Crossref, OpenAlex and arXiv and return a status report."""
title, doi = e.get("title", ""), re.sub(r"^(https?://(dx\.)?doi\.org/|doi:)", "", e.get("doi", ""), flags=re.I)
year = re.sub(r"\D", "", e.get("year", ""))[:4]
author = first_surname(e.get("author", ""))
res = {"key": e.get("_key", doi), "doi": doi, "title": title, "status": "", "problems": [], "suggestion": None}
if doi:
return _check_by_doi(res, title, doi, year, author)
if not title:
res["status"] = "unverifiable"
res["problems"].append("no DOI and no title")
return res
return _classify_match(res, _find_best(title, year, author), title, year, author)
def main() -> int:
"""Run the command-line interface."""
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("bib", nargs="?")
ap.add_argument("--doi", nargs="+", default=[])
ap.add_argument("--keys", help="comma-separated subset of citation keys")
ap.add_argument("--json", metavar="PATH", help="write the full report as JSON")
ap.add_argument("--delay", type=float, default=0.2, help="seconds between API calls (default 0.2)")
args = ap.parse_args()
if not args.bib and not args.doi:
ap.error("give a .bib file or --doi")
entries: list[dict] = [{"_key": d, "doi": d} for d in args.doi]
if args.bib:
try:
with open(args.bib, encoding="utf-8") as fh:
entries += parse_bib(fh.read())
except (OSError, ValueError, IndexError) as exc:
print(f"error: cannot read {args.bib}: {exc}", file=sys.stderr)
return 2
if args.keys:
wanted = set(args.keys.split(","))
entries = [e for e in entries if e["_key"] in wanted]
report = []
for e in entries:
try:
r = check_entry(e)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
r = {"key": e.get("_key"), "status": "error", "problems": [f"lookup failed: {exc}"], "suggestion": None}
report.append(r)
mark = {"verified": "OK ", "found-add-doi": "DOI+", "mismatch": "FAIL", "doi-not-found": "FAIL",
"not-found": "FAIL", "uncertain": "?? ", "unverifiable": "?? ", "error": "ERR "}[r["status"]]
print(f"{mark} {r['key']}: {r['status']}")
for p in r["problems"]:
print(f" - {p}")
if r.get("suggestion") and r["status"] == "found-add-doi":
print(f" + add doi = {{{r['suggestion']['doi']}}}")
time.sleep(args.delay)
if args.json:
with open(args.json, "w", encoding="utf-8") as fh:
json.dump(report, fh, indent=2, ensure_ascii=False)
bad = [r for r in report if r["status"] not in ("verified", "found-add-doi")]
print(f"\n{len(report)} checked: {len(report) - len(bad)} verified, {len(bad)} need attention")
return 1 if bad else 0
if __name__ == "__main__":
sys.exit(main())
SKILL.md
---
name: citation-verification
description: Verify that every reference in a manuscript really exists and matches its metadata, catching hallucinated, corrupted or mismatched citations before submission. Use when checking a .bib file or reference list, after an AI assistant drafted citations, when a DOI may point to the wrong paper, when adding missing DOIs, or before camera-ready. Queries Crossref, OpenAlex and arXiv; zero dependencies.
license: MIT
compatibility: Python 3.9+ standard library; needs network access to api.crossref.org, api.openalex.org and export.arxiv.org.
metadata:
version: "1.0"
category: literature-review
maintainer: Kalaris Labs
author: Kalaris Labs (Sayan Chowdhury)
tags: citations, hallucination, DOI, Crossref, OpenAlex, arXiv, references, research integrity
---
# Citation Verification
Language models (and tired humans) produce references that look right but are
not: plausible titles that were never published, real DOIs attached to the wrong
paper, wrong years, invented co-authors. Citing a paper that does not exist can
lead to desk rejection, a correction, or a misconduct inquiry. This skill makes
verification systematic.
## Non-negotiable rules
1. **Never write a reference from memory.** Every entry must come from a
retrievable record (Crossref, OpenAlex, arXiv, PubMed, publisher page).
2. **A reference is verified only when its identifier resolves to a record whose
title, first author and year all match.** A resolving DOI alone proves nothing.
3. **If you cannot verify a reference, say so.** Mark it `% UNVERIFIED` in the
`.bib`, tell the user, and never quietly keep it.
4. Verify the claim too: open the abstract or the cited section and confirm the
paper actually supports the sentence citing it.
## Workflow
```bash
export CROSSREF_MAILTO=you@university.edu # polite pool: faster, fewer rate limits
python scripts/verify_citations.py references.bib --json citation-report.json
python scripts/verify_citations.py --doi 10.1038/nature14539 10.1126/science.aaa8415
python scripts/verify_citations.py references.bib --keys smith2020,li2023 # re-check a subset
```
Each entry gets a status:
| Status | Meaning | Action |
|---|---|---|
| `verified` | Identifier resolves; title/author/year agree | None |
| `found-add-doi` | No DOI in the entry, but an unambiguous match exists | Add the suggested `doi` field |
| `mismatch` | DOI resolves to a *different* work, or year/author disagree | Fix from the record, or find the intended paper |
| `doi-not-found` | DOI does not resolve anywhere | Typo or fabricated: search by title |
| `uncertain` | Title matches but year/author disagree, or only a weak title match | Check manually (reprints, errata, homonymous papers) |
| `not-found` | No plausible match in Crossref, OpenAlex or arXiv | Treat as fabricated until proven otherwise (grey literature, theses and reports may legitimately be absent) |
Exit code `0` means everything is verified or only missing DOIs; `1` means something needs a human.
## Handling the hard cases
- **Conference papers without DOIs** (NeurIPS, ICML, ICLR, many CS venues): the script
falls back to arXiv. Cite the proceedings version with a `url` to the official
proceedings page, and optionally keep `eprint` for the arXiv ID.
- **Preprint vs published**: if both exist, cite the peer-reviewed version unless
the venue asks otherwise. OpenAlex links versions of the same work.
- **Books and chapters**: check ISBN via the publisher or Open Library. Crossref covers many but not all.
- **Datasets and software**: verify DataCite DOIs (`https://api.datacite.org/dois/{doi}`) or
Zenodo/Software Heritage identifiers.
- **Retractions**: check each DOI against Retraction Watch data (in Crossref as
`update-to` / `relation` metadata) and never cite a retracted paper as support.
- **Non-English or old literature**: absence from these indexes is common. Confirm via
library catalogs and document the source.
## Reporting to the user
Summarize as a table: key, status, problem, and the fix you applied or propose.
Apply only mechanical fixes automatically (adding a DOI for a `found-add-doi`
match). Anything `mismatch`, `uncertain` or `not-found` requires the user's
confirmation because only they know which paper they meant to cite.
## Related skills
- `bibtex-hygiene`: duplicates, missing fields, capitalization and formatting in the `.bib` file.
- `citation-management`: finding and exporting references from literature databases.
- `literature-review`: systematic searching and screening.