nature-downloader

어떤 스킬인가요?

Use when a user needs lawful academic full text, CNKI institutional access, English OA retrieval, publisher API access, institutional browser fallback, or supporting information downloads.

설치 방법

프로젝트 폴더에서 아래 명령어를 실행하고, 설치할 에이전트를 선택하세요.

설치 명령을 직접 실행해야 적용됩니다. 지원 에이전트와 필요한 권한·라이선스는 제작자의 안내를 확인하세요.

스킬 원문 파일

* text=auto
*.md text eol=lf
*.mjs text eol=lf
*.py text eol=lf
*.yaml text eol=lf
*.yml text eol=lf
*.json text eol=lf
*.tsv text eol=lf

__pycache__/
*.pyc
*.pyo
*.pdf
*.docx
*.xlsx
downloaded/
downloads/
extracted_text/
manifest.local.*
CX_tmp_*
.workbuddy/
.DS_Store

interface:
  display_name: "Nature Downloader"
  short_description: "Retrieve lawful full text and supporting information"
  default_prompt: "Use $nature-downloader to retrieve this paper and its requested supporting information through lawful access routes."

{
  "elsevier": {
    "name": "Elsevier / ScienceDirect",
    "config_url": "https://dev.elsevier.com/",
    "doi_pattern": "^10\\.(?:1016|1053|1067|1072|1097|1205|1383|1532|1588|1600|1618|2016|3168)/",
    "marker_pattern": "elsevier|sciencedirect",
    "credential_fields": ["api_key", "insttoken", "authtoken"]
  },
  "springer_nature": {
    "name": "Springer Nature",
    "config_url": "https://dev.springernature.com/docs/quick-start/api-access/",
    "doi_pattern": "^10\\.(?:1007|1023|1038|1057|1134|1186|1365)/",
    "marker_pattern": "springer|nature portfolio|biomed central|bmc\\b",
    "credential_fields": ["api_key"]
  },
  "ieee": {
    "name": "IEEE Xplore",
    "config_url": "https://developer.ieee.org/member/register",
    "doi_pattern": "^10\\.1109/",
    "marker_pattern": "\\bieee\\b|ieeexplore",
    "credential_fields": ["api_key", "fulltext_endpoint"]
  }
}

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "SchoolConfig",
  "description": "用户学校配置文件 schema，对应 ~/.config/lit-dl/school.json",
  "type": "object",
  "required": ["version", "school", "auth", "libraries"],
  "properties": {
    "version": {
      "type": "integer",
      "description": "配置版本号，用于后续迁移",
      "const": 1
    },
    "school": {
      "type": "object",
      "required": ["name"],
      "properties": {
        "name": {
          "type": "string",
          "minLength": 2,
          "description": "学校全称"
        },
        "code": {
          "type": ["string", "null"],
          "description": "学校代码/缩写"
        },
        "configured_at": {
          "type": ["string", "null"],
          "format": "date-time",
          "description": "配置时间 ISO 8601"
        },
        "source": {
          "type": ["string", "null"],
          "enum": ["preset", "manual", "resource_url", null],
          "description": "配置来源：资源入口链接 / 预设库 / 手动填写"
        }
      }
    },
    "auth": {
      "type": "object",
      "required": ["type", "sso_domain"],
      "properties": {
        "type": {
          "type": "string",
          "enum": ["cas", "sso", "custom"],
          "description": "认证类型"
        },
        "sso_domain": {
          "type": "string",
          "description": "统一身份认证域名，如 login.university.example"
        },
        "carsi_entry": {
          "type": ["string", "null"],
          "description": "CARSI 联邦认证入口 URL"
        },
        "carsi_sp_entity_id": {
          "type": ["string", "null"],
          "description": "CARSI SP EntityID（可选）"
        }
      }
    },
    "proxy": {
      "type": "object",
      "properties": {
        "type": {
          "type": ["string", "null"],
          "enum": ["ezproxy", null],
          "description": "代理类型"
        },
        "ezproxy_url": {
          "type": ["string", "null"],
          "description": "EZproxy 登录地址"
        }
      }
    },
    "libraries": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "minItems": 1,
      "description": "该校订阅的、走机构权限的数据库清单"
    },
    "discovery": {
      "type": "object",
      "properties": {
        "web_of_science_url": {
          "type": ["string", "null"],
          "description": "Web of Science / 学术资源文献聚合访问服务入口"
        },
        "resource_entry_url": {
          "type": ["string", "null"],
          "description": "用户提供的图书馆电子资源入口"
        },
        "resource_portal_url": {
          "type": ["string", "null"],
          "description": "资源聚合门户入口"
        },
        "auth_service_host": {
          "type": ["string", "null"],
          "description": "认证回跳服务域名"
        }
      },
      "additionalProperties": true
    },
    "notes": {
      "type": "string",
      "default": "",
      "description": "特殊提示"
    },
    "_warnings": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "校验时产生的警告（不阻塞写入）"
    }
  },
  "additionalProperties": false
}

# Intentionally empty.
#
# nature-downloader is distributed as an institution-neutral skill. Users must
# configure their own library/resource URL at runtime with:
#   python3 scripts/configure_school.py url "https://library.example.edu/resources"
version: 1
schools: []

# Nature Downloader 重构参考源码研究

日期：2026-07-15

## 研究范围与版本

本笔记只研究用户指定的两个一手源码仓库，不讨论 Nature Downloader 当前实现，也不修改现有代码。

- `ShZhao27208/Aut_Sci_Download`：研究提交 [`5e0de4c`](https://github.com/ShZhao27208/Aut_Sci_Download/tree/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2)（该提交日期 2026-06-29），重点为 CNKI/FSSO/WebVPN 路线。仓库采用 MIT License。[项目清单](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/pyproject.toml#L1-L12) [License](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/LICENSE#L1-L20)
- `kermitt2/article_dataset_builder`：研究提交 [`920fe29`](https://github.com/kermitt2/article_dataset_builder/tree/920fe298a81061e8a26d55d13829ccb6deb6501c)（该提交日期 2023-10-03），重点为 OA 发现、下载瀑布、恢复与输出。仓库采用 Apache-2.0 License，包版本为 0.2.6。[setup.py](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/setup.py#L3-L28) [License](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/LICENSE#L1-L12)

结论先行：前者适合参考“机构授权会话适配器”的边界；后者适合参考“OA 候选发现 + 强标识符归一化 + 状态持久化 + 多下载器回退”的管线思想。两者都不适合整段照搬。

## 一、Aut_Sci_Download 的 CNKI 路线

### 1. 认证与入口

仓库把 CNKI 机构访问抽象成两个模式：

1. `fsso`：用户先在 `https://fsso.cnki.net` 通过高校 SSO/CARSI 登录，再把浏览器 cookies 导出到本地 JSON；请求仍直连 `kns.cnki.net`。
2. `webvpn`：用户先登录学校 WebVPN，导出 cookies；之后将 CNKI URL 转换为学校 WebVPN 代理 URL。

README 明确要求用户自行登录并导出 cookies，cookie 文件默认位于 `~/.aut-sci-download/fsso_cookies.json` 或 `webvpn_cookies.json`。[认证说明](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/README.md#L135-L142) 运行时创建 `requests.Session`，设置浏览器 User-Agent、语言、可选代理，并把 JSON 中的 cookie name/value/domain/path 填入会话。[会话构造](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L70-L102)

WebVPN 不是浏览器自动化，而是把原 URL 的 hostname 用 AES-128-CFB 加密，再拼成学校代理 URL；学校 host、key、iv 来自配置/学校表。[URL 转换](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/webvpn_crypto.py#L14-L58) 两种模式共用后续搜索和下载逻辑，仅在 `_resolve_url` 这一层改变 URL。[模式适配](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L105-L116)

可借鉴：

- 将“用户完成机构登录”与“技能复用授权会话”分开；技能不收集用户名/密码。
- 用统一 `InstitutionSession`/`AccessTransport` 接口隔离 FSSO 直连和 WebVPN 改写，避免下载流程感知学校差异。
- 提供显式 `status/check`，在搜索/下载前给出 cookie 缺失、过期、WebVPN 未配置等可操作错误。

不能直接照搬：

- cookie 明文 JSON 与 WebVPN key/iv 明文配置没有权限收紧、系统钥匙串集成或自动轮换；重构时至少应限制本地文件权限，并明确 cookie 属于敏感凭据。
- `check_session` 只凭 HTTP 200 且正文包含 `cnki` 判为有效，可能把匿名页面、错误页误判成已获机构授权；登录重定向判断也只检查 URL 中少数字符串。[会话检查](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L119-L142)
- AES WebVPN URL 方案依赖具体厂商/学校实现，默认 `wrdvpnisthebest!` 不是通用协议；必须按学校能力探测或配置，不能声称覆盖所有高校。

### 2. 元数据识别与搜索入口

搜索流程先 GET CNKI 搜索页以建立/补充会话 cookie，然后向 `/kns8s/brief/grid` POST 一个 `QueryJson`，检索字段固定为主题 `SU`，语言为中文，页大小最多 50。[搜索请求](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L145-L222)

返回 HTML 后，主解析器从结果表抽取：题名、作者、来源、年份、详情 URL，并从 URL 查询参数中提取 `filename` 与 `dbcode`。结构化 CSS 选择器失败时，再用正则匹配旧/新详情链接，但回退结果会丢失作者、来源和年份。[结果解析](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L245-L335)

可借鉴：

- 不要只靠题名决定下载对象；保留 CNKI 自身的 `filename + dbcode + detail_url` 作为强定位信息。
- 搜索结果解析应有主解析器与兼容旧页面的回退解析器，并对“回退后字段不完整”显式打标。

不能直接照搬：

- `KuaKuCode`、`productStr`、CSS 选择器和 URL 格式都是页面内部细节，容易随 CNKI 改版失效，应该封装在可测试的 provider adapter 中，不应泄漏到总路由。
- 该实现没有 DOI、题名、作者、年份的消歧评分，也没有对多个同名结果进行确认；Nature Downloader 需要独立的规范化文献实体与置信度。
- `_extract_filename` 在没有 `filename` 时把 `v` 参数当 filename，这个假设未被验证，不宜作为可靠强标识符。[提取逻辑](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L317-L335)

### 3. 下载决策与失败回退

单篇下载先检查本地 `cnki_<id>.pdf` 是否存在且至少 10 KB；命中则直接返回缓存。否则访问新版 `kcms2` 详情页，若 HTTP 404 再试旧版 `kcms/detail/detail.aspx`。[详情页与缓存](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L338-L403)

下载链接按下列优先级从详情 HTML 中寻找：当前 `bar.cnki.net` PDF、经典 PDF 按钮、操作区 PDF、`bar.cnki.net` CAJ、经典 CAJ、任意 `bar.cnki.net order` 链接。[链接优先级](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L406-L459)

最终 GET 下载链接，检查状态码、文件头（PDF/CAJ）、疑似登录页和最小体积；CAJ 会改扩展名，成功结果返回文件路径、大小和格式。[文件下载](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/cnki_download.py#L462-L501)

可借鉴：

- 详情页新旧入口回退、PDF 优先/CAJ 次选、登录页检测、魔数校验、体积下限、幂等缓存，都是合理的 provider 内部策略。
- 下载结果应返回结构化 provenance：provider、访问模式、格式、字节数、最终文件路径、错误类型。

不能直接照搬：

- 当前校验存在明显漏洞：若内容既不是 PDF 也不是 CAJ，但大于 10 KB，代码仍会写成 `.pdf` 并标记成功；Nature Downloader 必须将“HTTP 成功”与“文件有效”严格分开。
- 缓存只按路径和 10 KB 判定，不重新验证 magic/header，也没有元数据 sidecar；可能把旧错误页永久当缓存。
- 链接提取依赖具体 DOM 文本和内部域名，没有处理 JavaScript 生成链接、验证码、下载额度、并发限制、授权范围不足等状态。
- 全量 `resp.content` 一次读入内存，不适合大文件；应流式写临时文件，校验后原子改名。

### 4. 输出与依赖

默认输出目录为 `~/papers`，配置目录为 `~/.aut-sci-download`；项目只声明 `requests`、`pycryptodome`、`beautifulsoup4` 三个核心依赖。[配置默认值](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/config.py#L16-L18) [依赖](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/pyproject.toml#L7-L12)

其配置层把 API key 放 `.env`，普通配置放 `config.json`，这是“敏感配置与普通配置分离”的可借鉴雏形；但文件仍是普通明文文件，没有权限或密钥存储机制。[配置写入](https://github.com/ShZhao27208/Aut_Sci_Download/blob/5e0de4cf68ab48dce8763fe7ca76a6f29874afc2/scripts/config.py#L85-L143)

## 二、article_dataset_builder 的 OA 路线

### 1. 定位、入口与认证

该项目不是面向一次对话下载一篇论文的工具，而是面向 DOI/PMID/PMCID/CORD-19 清单的批量 OA 数据集采集器；它同时做元数据聚合、PDF 获取、可选 TEI 转换、缩略图和 S3 输出。[项目定位](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/Readme.md#L4-L38)

入口是本地文件清单：每行一个 DOI、PMID 或 PMCID，或 CORD-19 CSV；不是自然语言、题名搜索或浏览器会话。[输入类型](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/Readme.md#L8-L13) 外部服务认证主要是 Unpaywall/Crossref 的联系邮箱，以及可选 S3 凭据；OA 发现本身不使用出版商订阅凭据。[配置样例](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/config.json)

对 Nature Downloader 的意义：只应借鉴 OA 子系统，不应把整个数据集构建器嵌入交互式 skill，也不能用它替代非 OA 出版商 API 或用户机构授权。

### 2. 元数据识别与归一化

DOI 会去除两种常见 DOI URL 前缀、去空白并转小写。[DOI 清洗](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1368-L1373) 元数据先请求 biblio-glutton；若 DOI 查询失败，再请求 Crossref，并删除体积很大的参考文献列表。[元数据查找](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L286-L336)

每个条目获得内部 UUID，并在 LMDB 中建立 DOI/PMID/PMCID/内部 ID 到同一条目的映射；元数据记录同时保存阶段状态，如 `has_valid_oa_url`、`has_valid_pdf`、`has_valid_tei`。[标识符映射](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L737-L804) [状态字段](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1441-L1453)

可借鉴：

- 先建立统一文献实体，再运行下载；输入标识符、规范 DOI、PMID/PMCID、出版商标识符都映射到同一内部记录。
- 将“识别到 OA URL”“下载到有效 PDF”“补充材料已下载”等阶段拆成独立状态，支持幂等恢复。
- 元数据源与全文源解耦：Crossref/聚合器回答“它是谁”，Unpaywall/PMC 回答“OA 文件在哪”。

不能直接照搬：

- biblio-glutton 默认公共实例的可靠性连 README 自己也提示不足；它不是 Nature Downloader 必需依赖。[服务说明](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/Readme.md#L94-L108)
- README 称 biblio-glutton 未配置时只用 Crossref，但代码在配置为空时立即返回 `None`，与说明不一致；需自行实现并测试元数据回退。
- `getUUIDByStrongIdentifier` 返回 LMDB 原始 bytes，而调用方后续把它当字符串 `.encode()`，现有“恢复已处理条目”路径存在类型风险。[查重调用](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L558-L567) [返回实现](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1133-L1140)

### 3. OA 下载决策

项目实际采用的是候选瀑布，而不是简单“调用 Unpaywall 即结束”：

1. 特定 Elsevier COVID OA 本地镜像映射（DOI/PII）。
2. 旧数据目录中已存在的有效 PDF。
3. 若有 PMCID，优先查 PMC OA 文件清单并使用 NIH FTP archive。
4. 若有 DOI，实时调用 Unpaywall；优先 `best_oa_location.url_for_pdf`，再尝试 PMC 特例与其他 `oa_locations`。
5. 若仍无结果，使用元数据聚合记录中已有的 `oaLink`。

主决策链见 [`processTask`](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L889-L945)，Unpaywall 候选选择见 [`unpaywalling_doi`](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L207-L234)。PMC 官方 OA file list 会在首次运行时下载并建立本地 LMDB 索引，后续 PMCID 可直接映射到 archive URL。[PMC 索引](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L157-L205)

可借鉴：

- OA 判断不应由 DOI 前缀或出版商名称推断，而应基于实时 OA location 数据与可信仓储映射。
- 同一论文可有多个合法 OA 候选，应保留候选列表、host/repository、版本、license、是否直链 PDF 等信息，再评分选择。
- 对 PMCID 优先 PMC archive 很有价值，因为 archive 可能同时含 PDF 与 JATS/NXML；这也为补充材料/结构化全文提供清晰扩展点。

不能直接照搬：

- 代码只返回第一个满足启发式的 URL，没有保留所有候选、版本（published/accepted/submitted）、host 类型、license 或选择原因。
- `best_oa_location` 可能为空，但后续 `elif` 仍直接访问它；调用者用裸 `except` 吞掉异常，诊断信息会丢失。
- 项目声称 fair-use 覆盖不可再分发文章，但“可下载”不等于“允许再分发”；Nature Downloader 必须把个人获取、OA 状态与再分发许可分开记录，不能沿用模糊语义。[项目声明](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/Readme.md#L15-L15)

### 4. 下载器回退与有效性验证

对一个已选 URL，项目依次尝试：FTP 时先 `wget`、再 urllib FTP；然后 `cloudscraper`、`requests`，最后非 FTP 再 `wget`。[下载瀑布](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1468-L1492) 下载后用 `python-magic` 检查 MIME 是否为 `application/pdf`，不只看扩展名。[有效性检查](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1425-L1439) 对 PMC `.tar.gz` 会提取第一个 PDF 与 NXML，并清理 archive。[PMC archive](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1617-L1675)

可借鉴：

- 将“候选 URL 选择”和“传输实现回退”拆成两层；同一 OA URL 可按普通 HTTP、特殊站点适配器、浏览器访问等方式尝试。
- 每次传输必须落临时文件，使用内容/MIME 校验，失败时清理；PMC archive 单独作为资源包类型处理。
- 失败状态持久化后可显式重试，而不是每次从头重复元数据解析和所有候选。[失败重跑](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1328-L1359)

不能直接照搬：

- 大量请求使用 `verify=False`，不应复制。
- `wget` 通过字符串拼接 URL 并 `shell=True` 执行，存在命令注入面；重构应使用参数数组或纯库调用。[wget 实现](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1530-L1583)
- cloudscraper 的重定向递归调用漏传 `filename`，该回退分支本身有缺陷。[cloudscraper 实现](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1494-L1528)
- Cloudflare 绕过、随机 User-Agent 与无差别重试不是通用 OA 策略；应遵守站点条款、限速和 robots/API 规范，并把需要真实用户会话的情况交给受控 web access。

### 5. 状态、输出与依赖

项目用两套 LMDB 保存条目和标识符映射，以 `batch_size` 控制线程池批次；README 警告并发过高可能导致 OA 站点封锁。[批处理](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L530-L575) [并发说明](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/Readme.md#L110-L110)

每条记录生成 UUID 命名的 PDF、JSON，可选 NXML、TEI、缩略图和标注；本地存储按 UUID 分层目录，也可上传 S3。另可输出逐行 JSON 元数据 dump 和 `map.json` catalogue。[单条输出](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1024-L1038) [文件管理](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L1041-L1131) [catalogue](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/article_dataset_builder/harvest.py#L396-L437)

核心 Python 依赖为 boto3、python-magic、lmdb、tqdm、requests、cloudscraper、BeautifulSoup；可选处理还依赖系统 `wget`、ImageMagick、GROBID、Pub2TEI。[依赖](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/setup.py#L14-L28) [系统依赖](https://github.com/kermitt2/article_dataset_builder/blob/920fe298a81061e8a26d55d13829ccb6deb6501c/Readme.md#L40-L57)

对于交互式 Nature Downloader，应借鉴状态模型和 provenance sidecar，而不是引入整套 LMDB/S3/GROBID 数据集设施。单篇下载可用轻量 manifest；批量模式再考虑 SQLite/队列。

## 三、建议提炼出的统一设计原则

### 1. 路由与 provider 分层

建议总路由只做分类，不包含站点细节：

```text
规范化请求
  -> 识别中文/英文与强标识符
  -> 中文：CNKI institution provider
  -> 英文：OA resolver
       -> 确认 OA：OA candidate pipeline
       -> 非 OA：publisher credential provider / web access
  -> 用户明确要求 SI：supplementary-material pipeline
```

CNKI 的 URL 加密、DOM 选择器，OA 的 Unpaywall/PMC 特例，都应留在各自 provider 内部。

### 2. 统一下载记录

两仓库共同暴露出“只返回文件路径不够”的问题。建议每篇论文生成一个 manifest，至少包括：

- 原始请求、规范 DOI/PMID/PMCID/CNKI filename/dbcode；
- 语言、OA 判断结论与证据时间；
- 选中的 provider、候选 URL、最终 URL、访问模式（公开 OA/机构授权/API/web access）；
- 文件 SHA-256、MIME、字节数、PDF/CAJ/资源包类型；
- license/版本信息（若来源提供）；
- SI 是否由用户明确请求、发现了哪些附件、下载了哪些；
- 每次失败的 typed error 与下一可执行动作。

### 3. 应保留的失败类别

- `credentials_missing`：需要 API key/token 或机构 cookie；
- `session_expired`：机构会话失效；
- `not_entitled`：登录有效但机构无该资源权限；
- `oa_not_found`：未找到公开 OA 候选，不等于论文不存在；
- `metadata_ambiguous`：题名/作者匹配不唯一；
- `download_link_changed`：页面存在但解析器找不到链接；
- `challenge_required`：验证码/交互式挑战，转 web access；
- `invalid_content`：HTTP 200 但内容不是目标文件；
- `rate_limited`：需要退避，而不是立刻切换随机 User-Agent；
- `si_not_requested`：发现 SI 但根据产品规则不下载。

## 四、最终取舍

| 机制 | 借鉴程度 | 理由 |
|---|---:|---|
| CNKI FSSO/WebVPN 统一 transport seam | 高 | 将机构授权差异隔离于下载流程之外 |
| 用户自行登录、技能复用 cookie | 中高 | 不接触密码；但必须加强凭据存储与会话验证 |
| CNKI 新旧详情页与 PDF/CAJ 优先级 | 中 | 可作为 provider 内部初版，需持续测试与更严格校验 |
| 通过中文关键词直接认定目标文献 | 低 | 缺少消歧与强标识符确认 |
| Unpaywall + PMC +其他 OA location 候选瀑布 | 高 | 是 OA 发现的正确基本方向 |
| LMDB 强标识符映射和阶段状态 | 中高 | 思想好；交互式 skill 可先用更轻的 manifest/SQLite |
| 多 transport 回退 + MIME 验证 | 高 | 但实现应重写，不能复制 `verify=False`、shell wget 等细节 |
| 整套 article_dataset_builder 作为依赖 | 低 | 过重、较旧、面向数据集构建，且存在已识别实现缺陷 |
| 默认下载/处理 SI | 不采用 | 新产品规则是只有用户明确要求时才进入 SI pipeline |

最重要的边界是：`article_dataset_builder` 提供的是 OA 采集思想，不提供“非 OA 出版商 API 下载”能力；`Aut_Sci_Download` 的 CNKI 路线提供的是会话复用范例，不证明其对所有学校、所有 CNKI 页面或所有授权状态都可靠。

# Nature Downloader Merge Implementation Plan

> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.

**Goal:** Rebuild `nature-downloader` as a complete skill by combining the GitHub repository's real browser-based PDF downloader with the local multi-school configuration wizard.

**Architecture:** Import the upstream repository as the base. Add Python configuration modules under `src/`, data files under `data/`, and a small CLI wrapper that can configure schools, run health checks, and then invoke the existing Node download path. Keep credential handling in the browser only.

**Tech Stack:** Node.js ES modules for CDP/browser PDF download, Python 3 for school configuration and PDF text extraction, JSON/YAML config files, shell-level verification.

---

### Task 1: Import Upstream Repository

**Files:**
- Modify: workspace root

- [ ] Fetch `Flyme886/nature-downloader` into a temporary directory.
- [ ] Copy tracked upstream files into the workspace without overwriting `.git`.
- [ ] Verify that `README.md`, `SKILL.md`, and `scripts/` exist.

### Task 2: Merge Local Configuration Wizard

**Files:**
- Create: `src/config.py`
- Create: `src/wizard.py`
- Create: `src/validators.py`
- Create: `src/health_check.py`
- Create: `src/schools_loader.py`
- Create: `data/schools.yaml`
- Create: `data/school.schema.json`
- Create: `scripts/configure_school.py`
- Create: `tests/python/test_config_wizard.py`

- [ ] Copy local configuration modules and data files.
- [ ] Add a CLI wrapper for preset configuration, health check, and config display.
- [ ] Add Python tests using a temporary `LIT_DL_CONFIG_DIR`.
- [ ] Run Python tests and fix failures.

### Task 3: Bridge Config To Download Workflow

**Files:**
- Modify: `scripts/batch_download.mjs`
- Create: `scripts/lib/school-config.mjs`
- Modify: `scripts/lib/status-codes.mjs` if needed
- Create or modify: `tests/unit/school-config.test.mjs`

- [ ] Load `~/.config/lit-dl/school.json` or `LIT_DL_CONFIG_DIR/school.json`.
- [ ] Prefer configured discovery URL when present; fall back to the current Web of Science URL.
- [ ] Preserve the existing SJTU download path and status behavior.
- [ ] Add Node tests for config discovery and fallback behavior.

### Task 4: Update Skill Documentation

**Files:**
- Modify: `README.md`
- Modify: `SKILL.md`

- [ ] Document first-run school configuration.
- [ ] Document the implemented download route and current limitations.
- [ ] Keep safety boundaries explicit: no credential scraping, no CAPTCHA bypass, no unauthorized mirrors.

### Task 5: Verify

**Commands:**
- `python3 -m unittest discover -s tests/python`
- `node --check scripts/batch_download.mjs`
- `node --check scripts/browser_pdf_downloader.mjs`
- `node --test tests/unit/*.test.mjs`

- [ ] Run all commands fresh.
- [ ] Report exact pass/fail status and any remaining limitations.

id	project	title	doi	year	venue	publisher	failure_stage	status	source_url	current_url	next_action	notes
1	example-project	Example article title	10.xxxx/example	2026	Example Journal	Example Publisher	publisher_redirect	carsi_waiting_user	https://example.com/article	https://login.university.example/	user_complete_institution_login_in_chrome	Complete institutional login in Chrome, then retry the same tab.

id	title	doi	year	venue	status	pdf_path	si_status	si_paths	source_url	notes
1	Innovative Use of an Injectable, Self-Healing Drug-Loaded Pectin-Based Hydrogel for Micro- and Supermicro-Vascular Anastomoses	10.1021/acs.biomac.4c00102	2024	Biomacromolecules	downloaded	PDFs/Kocaaga_2024_Biomacromolecules_vascular_anastomoses.pdf	downloaded	SupportingInformation/Kocaaga_2024_Biomacromolecules_vascular_anastomoses_SI.pdf	https://pubs.acs.org/doi/pdf/10.1021/acs.biomac.4c00102	verified test case

id	project	title	doi	year	venue	publisher	status	source_url	current_url	next_action	notes
1	example-project	Example ScienceDirect article	10.xxxx/example	2026	Example Journal	ScienceDirect	sciencedirect_robot_check	https://www.sciencedirect.com/science/article/pii/...	https://www.sciencedirect.com/...	user_complete_publisher_verification	Do not auto-click the challenge; continue from the same tab after user verification.

MIT License

Copyright (c) 2026 baihe26

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

name: nature-downloader
version: 2.1.0
description: >
  Download academic full text through publisher APIs, lawful open-access
  fallbacks, CNKI, or a user's institution-authorized browser session, with
  secure credential storage and an auditable per-paper manifest.

always_load:
  - SKILL.md

references:
  on_demand:
    - condition: user needs first-run school/library resource configuration
      path: README.md
    - condition: configuring known school presets, resource URL inference, or health checks
      path: scripts/configure_school.py
    - condition: configuring or validating Elsevier, Springer Nature, or IEEE API credentials
      path: scripts/configure_credentials.py
    - condition: batch DOI/title/topic downloads through Web of Science or open-access sources
      path: scripts/batch_download.mjs
    - condition: publisher API routing, OA fallback, and provider-specific full-text status
      path: scripts/lib/publisher-providers.mjs
    - condition: browser-authenticated PDF fetching and PDF signature checks
      path: scripts/browser_pdf_downloader.mjs
    - condition: canonical download statuses and manifest status semantics
      path: scripts/lib/status-codes.mjs
    - condition: lawful OA and applicable publisher APIs are exhausted and the task needs Web of Science, an institution-authorized browser session, publisher verification, authentication handoff, or browser-context PDF transfer
      path: references/institutional-browser-workflow.md
    - condition: Supporting Information was requested, downloaded files need final verification and naming, or an access attempt reaches a typed failure or retry state
      path: references/delivery-verification-and-failures.md

# `nature-downloader` Skill

[中文说明](README.md)

`nature-downloader` obtains paper full text, PDFs, HTML/XML full text, or auditable download status through publisher APIs, lawful open-access routes, CNKI, or the user's own institution-authorized access.

## Workflow

1. First identify the paper language, publisher, and whether supplementary information is needed.
2. Prefer lawful publisher APIs when they are available; fall back to OA, repository, or institution-authorized access only when needed.
3. Record the access path, failure reason, and reusable configuration so batch jobs remain auditable.
4. Never bypass paywalls, DRM, CAPTCHAs, or two-factor verification, and never read or export browser cookies, passwords, localStorage, or session files.

## Supporting Information Confirmation

Every download command must choose one SI mode explicitly:

```bash
--no-si  # download the main text only
--si     # download the main text and available Supporting Information
```

If neither option is provided, the CLI returns `si_confirmation_required` and does not create output directories or download files. Passing both options fails argument validation. In batch jobs, the choice applies to the whole batch.

## Configuration

### Library and CNKI

Save the resource portal that the user actually uses:

```bash
python3 scripts/configure_school.py infer "https://example.edu/library/resources"
python3 scripts/configure_school.py url "https://example.edu/library/resources"
python3 scripts/configure_school.py show
python3 scripts/configure_school.py health --force
```

Regular school configuration is stored in `~/.config/lit-dl/school.json`.

### Publisher APIs

Configure publisher APIs only when non-OA English papers match those publishers:

- Elsevier: [Developer Portal](https://dev.elsevier.com/)
- Springer Nature: [API Access](https://dev.springernature.com/docs/quick-start/api-access/)
- IEEE: [Developer Registration](https://developer.ieee.org/member/register)

Store API keys through hidden input:

```bash
python3 scripts/configure_credentials.py set elsevier
python3 scripts/configure_credentials.py set springer_nature
python3 scripts/configure_credentials.py set ieee --fulltext-endpoint 'https://issued-endpoint.example/articles/{doi}'
python3 scripts/configure_credentials.py show
python3 scripts/configure_credentials.py validate elsevier
python3 scripts/configure_credentials.py delete elsevier
```

When the user has already provided a publisher API key in the conversation, the agent should save it through standard input without echoing it in commands, replies, or manifests:

```bash
python3 scripts/configure_credentials.py set elsevier --stdin
```

Otherwise prefer local hidden input. Institutional passwords, OTP codes, cookies, and session tokens are not handled by this rule. Secrets are stored in `~/.config/lit-dl/credentials.json` with owner-only permissions and are displayed only by their last four characters.

Unpaywall needs a compliant contact email:

```bash
python3 scripts/configure_credentials.py contact-email researcher@example.org
```

The email is stored as non-secret settings in `~/.config/lit-dl/settings.json`.

## Download Examples

Download main PDFs by DOI:

```bash
node scripts/batch_download.mjs \
  --dois "10.1007/s00122-021-03957-1,10.1111/pbi.14066" \
  --no-si \
  --out "./literature-downloads"
```

Chinese titles use the CNKI route only:

```bash
node scripts/batch_download.mjs \
  --title "乡村振兴背景下数字治理研究" \
  --no-si \
  --out "./literature-downloads"
```

For an exact English OA title:

```bash
node scripts/batch_download.mjs \
  --title "Attention Is All You Need" \
  --open-access \
  --no-si \
  --out "./literature-downloads"
```

Search by topic and download SI:

```bash
node scripts/batch_download.mjs \
  --topic "rice blast resistance gene" \
  --count 10 \
  --si \
  --out "./literature-downloads"
```

For a known lawful full-text URL:

```bash
node scripts/batch_download.mjs \
  --pdf-url "https://arxiv.org/pdf/1706.03762" \
  --title "Attention Is All You Need" \
  --no-si \
  --out "./literature-downloads"
```

Use `--language zh|en` or `--route cnki|open_access|elsevier|springer_nature|ieee|web_access` to override ambiguous metadata. A `--source-url` pointing to CNKI forces the Chinese CNKI route.

## API Failure and Web Access Fallback

When Elsevier, Springer Nature, or IEEE APIs return no entitlement or no full text, the downloader tries PMC, Unpaywall, publisher OA, and lawful repositories first. Only when API and OA routes fail does it return `api_fallback_confirmation_required`; after confirmation, rerun with the publisher-specific choice:

```bash
--api-fallback-web-for elsevier
--no-api-fallback-web-for springer_nature
```

For the whole batch, use `--api-fallback-web` or `--no-api-fallback-web`. Web Access reuses the user's logged-in Chrome institutional session; login, QR approval, OTP, and complex verification remain user actions.

## Typical Requests

- "Configure my university library entry point so future paper downloads can reuse it."
- "Use my logged-in Chrome session to download PDFs for these DOIs into the current project."
- "Use the authorized CNKI route for this Chinese paper; save it if possible and explain why if not."

## What You Need To Provide

- DOI, title, paper page link, or Chinese paper title.
- Library database entry point or logged-in Chrome session.
- Target output directory and naming preference.
- Whether Supporting Information should also be downloaded; it is off by default.

## Outputs

- Local PDF, HTML, or text file.
- Access path, OA/API fallback history, save path, integrity hash, and failure reason for each paper in `manifest.json`.
- Reusable school-entry configuration, usually at `~/.config/lit-dl/school.json`.

## Runtime and Dependencies

- First-time setup can use `scripts/configure_school.py` to identify and save resource entry points.
- Publisher API keys are configured with `scripts/configure_credentials.py`, stored separately with owner-only permissions, and never written to manifests.
- Real downloading depends on local browser login state and available web-access / CDP control.
- Chinese papers default to the user's authorized CNKI or library CNKI entry point.

## Boundaries

- The skill does not bypass paywalls, use mirror sites, or read/export cookies, passwords, localStorage, or session files.
- For a visible slider, checkbox, robot check, or simple verification button, it first makes at most two bounded attempts in the same authenticated browser tab and continues when the page confirms success.
- User action is required only after those attempts fail or for image selection, QR approval, SMS/OTP, passkeys, hardware keys, or two-factor authentication.
- Without legal access, the skill only reports status and possible alternatives.

## Related Skills

- `nature-reader`: turn obtained PDF/HTML into a full-paper reader.
- `nature-academic-search`: find target papers from title, DOI, or topic.

# nature-downloader

<p align="center">
  <img src="assets/banner.jpg" alt="nature-downloader — 合法 OA、出版商 API 与机构授权全文下载" width="100%">
</p>

`nature-downloader` 按文献语言、出版商和可用凭据自动选择合法下载路线：

```text
确认是否下载 SI
→ 中文文献：只走 CNKI / 知网机构授权
→ 英文文献
   ├─ Elsevier / Springer Nature / IEEE 且有可用 Key
   │  ├─ 出版商 API 成功：结束，不强制判断 OA
   │  └─ API 失败：PMC → Unpaywall → 出版商 OA / 合法仓储
   └─ 其他出版商：先查 OA，OA 不可用时走 Web Access 机构授权
```

## 工作方式

1. 先判断文献语言、出版商和 SI 需求，再决定路由。
2. 能走合法 API 就优先走 API；API 不通时再降级到 OA / 仓储 / 机构授权路径。
3. 输出时同时记录访问路径、失败原因和可复用配置，方便后续批量任务。
4. 全程不绕过付费墙、DRM、验证码或双重认证，也不读取或导出浏览器 cookie、密码、localStorage 或 session 文件。

## 下载前必须确认 SI

所有下载命令必须显式选择一次：

```bash
--no-si  # 只下载正文
--si     # 下载正文和可找到的 Supporting Information
```

两者均未提供时，脚本返回 `si_confirmation_required`，且不会创建输出目录或下载文件。两者同时提供时参数校验失败。批量任务的选择作用于整个批次。

## 配置

### 图书馆与 CNKI

优先保存用户实际使用的图书馆资源入口：

```bash
python3 scripts/configure_school.py infer "https://example.edu/library/resources"
python3 scripts/configure_school.py url "https://example.edu/library/resources"
python3 scripts/configure_school.py show
python3 scripts/configure_school.py health --force
```

普通配置保存在 `~/.config/lit-dl/school.json`。

### 出版商 API

非 OA 英文文献命中对应出版商时才需要配置：

- Elsevier：[Developer Portal](https://dev.elsevier.com/)
- Springer Nature：[API Access](https://dev.springernature.com/docs/quick-start/api-access/)
- IEEE：[Developer Registration](https://developer.ieee.org/member/register)

使用隐藏输入保存 API key：

```bash
python3 scripts/configure_credentials.py set elsevier
python3 scripts/configure_credentials.py set springer_nature
python3 scripts/configure_credentials.py set ieee --fulltext-endpoint 'https://issued-endpoint.example/articles/{doi}'
python3 scripts/configure_credentials.py show
python3 scripts/configure_credentials.py validate elsevier
python3 scripts/configure_credentials.py delete elsevier
```

当用户已经主动在对话中提供出版商 API key 时，agent 应直接使用标准输入安全保存，不要求重新生成，也不在命令参数、回复或 manifest 中回显：

```bash
python3 scripts/configure_credentials.py set elsevier --stdin
```

未主动提供时仍优先使用本地隐藏输入。机构密码、OTP、Cookie 和会话令牌不适用此规则。

Elsevier 获得机构 token 时可额外传入 `--insttoken` 或 `--authtoken`。IEEE 普通 Metadata API key 不代表收费全文权限；只有获得 Full-Text Access API 产品后，才配置由 IEEE 提供的 endpoint 模板。秘密保存在 `~/.config/lit-dl/credentials.json`，文件权限为 `0600`，展示时只显示末四位。

Unpaywall 需要合规联系邮箱：

```bash
python3 scripts/configure_credentials.py contact-email researcher@example.org
```

该邮箱保存在非秘密配置 `~/.config/lit-dl/settings.json`。

## 下载示例

按 DOI 下载正文：

```bash
node scripts/batch_download.mjs \
  --dois "10.1007/s00122-021-03957-1,10.1111/pbi.14066" \
  --no-si \
  --out "./文献自动下载"
```

中文题名只走知网：

```bash
node scripts/batch_download.mjs \
  --title "乡村振兴背景下数字治理研究" \
  --no-si \
  --out "./文献自动下载"
```

默认 PDF 优先、允许 CAJ；只接受 PDF 时增加 `--cnki-format pdf`。学校提供专用知网入口时增加 `--cnki-url URL`。

英文 OA 精确题名：

```bash
node scripts/batch_download.mjs \
  --title "Attention Is All You Need" \
  --open-access \
  --no-si \
  --out "./文献自动下载"
```

主题检索并下载 SI：

```bash
node scripts/batch_download.mjs \
  --topic "rice blast resistance gene" \
  --count 10 \
  --si \
  --out "./文献自动下载"
```

已知合法全文 URL：

```bash
node scripts/batch_download.mjs \
  --pdf-url "https://arxiv.org/pdf/1706.03762" \
  --title "Attention Is All You Need" \
  --no-si \
  --out "./文献自动下载"
```

元数据冲突时可用 `--language zh|en` 或 `--route cnki|open_access|elsevier|springer_nature|ieee|web_access` 覆盖。`--source-url` 指向 CNKI 时强制中文 CNKI 路由。

## API 失败与 Web Access 回退

三家 API 返回无 entitlement 或无全文时，会先自动尝试 PMC、Unpaywall、出版商 OA 和合法仓储。只有 API 与 OA 都未取得全文时，才返回 `api_fallback_confirmation_required`；确认后按出版商重新运行：

```bash
--api-fallback-web-for elsevier
--no-api-fallback-web-for springer_nature
```

全批次统一选择也可使用 `--api-fallback-web` 或 `--no-api-fallback-web`。Web Access 复用用户已登录的 Chrome 机构会话；登录、QR、OTP 和复杂验证仍由用户本人完成。

如果 PMC/Unpaywall 等 OA 检查无法确认文章状态，manifest 会记录 OA assessment 为 `unknown`，但不会把它误标为非 OA；后续仍可使用机构 Web Access 寻找授权全文。

## 输出

- 本地 PDF、HTML、文本或 CNKI 原生全文文件。
- `manifest.json` 记录每篇文献的访问路径、OA/API 回退历史、保存路径、完整性哈希、SI 选择和失败原因。
- 可复用学校入口配置，通常保存在 `~/.config/lit-dl/school.json`。

```text
文献自动下载/
  PDFs/
  FullText/
  CNKI/
  SupportingInformation/
  manifest.json
```

`manifest.json` 记录规范 DOI、语言、出版商、路由、OA 证据、访问模式、正文格式、MIME、大小、SHA-256、SI 选择和失败原因，并递归移除 API key、token、cookie 等秘密字段。

正文成功格式包括：

- PDF：`downloaded` / `open_access_downloaded`
- CNKI CAJ、Springer JATS/XML：`native_fulltext_downloaded`
- 可读 HTML 全文：`full_text_html_available`
- 正文和 SI：`downloaded_with_si`

## 边界

- 不绕过付费墙，不使用镜像站，也不读取或导出 cookie、密码、localStorage 或 session 文件。
- 对可见滑块、复选框、机器人校验或简单确认按钮，只在同一已授权浏览器标签页中最多尝试两次，并在页面确认成功后继续。
- 图片选择、二维码确认、短信/OTP、通行密钥、硬件密钥或双重认证必须由用户本人完成。
- 没有合法访问权限时，只报告状态、失败原因和可能替代路线。

## 相关技能

- `nature-reader`：把已获取的 PDF/HTML 转成全文阅读材料。
- `nature-academic-search`：从题名、DOI 或主题查找目标文献。

## 验证

```bash
python3 -m unittest discover -s tests/python
node --test tests/unit/*.test.mjs
node --check scripts/batch_download.mjs
node --check scripts/browser_pdf_downloader.mjs
```

# Delivery, Verification, and Failure Handling

## Contents

- [Supporting Information](#supporting-information)
- [Verification and Reading](#verification-and-reading)
- [Zotero](#zotero)
- [Naming Convention](#naming-convention)
- [Failure Handling](#failure-handling)
- [To Confirm With The User on First Run](#to-confirm-with-the-user-on-first-run)


Load this reference when the user requests Supporting Information, when files are ready for
verification and delivery, or when an access attempt reaches a typed failure or retry state.

## Supporting Information

**Always confirm SI before file download.** Fetch SI only when the user explicitly chooses it (e.g. "连补充材料一起下", "include SI", "download supplementary", "把补充材料也下了"). When the user chooses no, pass `--no-si` and do not perform extra attachment navigation.

When the user does ask for supporting information, use this method:

1. Open the article landing page, not only the PDF page.
2. Extract all links with text or href matching:
   - `Supporting Information`
   - `Supplementary`
   - `Supplemental`
   - `/doi/suppl/`
   - `/suppl_file/`
   - `_si_`
   - `mmc1`, `mmc2` (Elsevier/ScienceDirect supplement pattern)
3. Download every PDF/DOCX/XLSX/video/data file that is clearly a legitimate supplement, using the browser context if needed.

For the WoS batch route, an explicit SI request maps to `--si`. When an exact title is known, pass it as both `--topic` and `--title` with `--count 1`. WoS + `--si` must:

- keep each paper in its own readable-title folder;
- place only the verified main PDF and clearly labelled SI files in that folder;
- preserve original attachment names when available;
- follow a supplementary landing page at most one level deep;
- exclude external repository links such as GitHub, Zenodo, Figshare, Dryad, and OSF;
- keep the main PDF and report `si.status = not_found` when no SI exists;
- report `partial` when some SI files fail without treating the main PDF as failed.

Do not apply the clean per-article folder behavior to CNKI, `--open-access`, bare `--pdf-url`, or direct `--dois` routes.

ACS fallback pattern, only after verifying the DOI and article page:

```text
https://pubs.acs.org/doi/suppl/<DOI>/suppl_file/<journal-code>_si_001.pdf
```

Do not invent supplement URLs as facts. If a guessed URL returns 404, record "not found" and inspect the article page.

## Verification and Reading

After downloading, verify every file.

For PDFs:

```powershell
$env:PYTHONUTF8='1'
python -X utf8 "$env:USERPROFILE\.claude\skills\nature-downloader\scripts\extract_pdf_text.py" `
  --pdf "D:\path\paper.pdf" `
  --pages 3
```

This should report page count and extracted text. The script also reconfigures stdout/stderr to UTF-8 internally to reduce Windows GBK failures. If extraction fails but the PDF is valid, try PyMuPDF, OCR, or the local `pdf` skill.

Minimum verification checklist:

- File exists and size is plausible.
- First bytes are `%PDF` for PDF files.
- Page count is nonzero.
- Extracted text includes the article title, abstract, or supporting information title.
- For HTML full text, saved HTML/text includes the article title or DOI, and the user-facing reply states that no valid PDF was available.
- Save a small manifest with DOI, title, source URL, download date, and supplement status when doing more than one paper.

## Zotero

Zotero import is useful for metadata, DOI, citation keys, and library organization, but it does not replace local PDF verification. If Zotero imports a paper, still check whether the PDF attachment is present and readable. If the user wants a project folder with full text, save PDFs explicitly to that folder.

## Naming Convention

Use readable filenames:

```text
FirstAuthor_Year_Journal_short-title.pdf
FirstAuthor_Year_Journal_short-title_SI.pdf
```

For project work, keep a folder like:

```text
文献自动下载/
  manifest.tsv
  PDFs/
  SupportingInformation/
  extracted_text/
```

## Failure Handling

If direct publisher navigation triggers ScienceDirect "Are you a robot?", Cloudflare, CAPTCHA, or another bot challenge:

- First, attempt automatic verification via `scripts/lib/anti-bot.mjs`.
- If auto-verification succeeds, continue the download normally.
- If auto-verification fails, record `verification_auto_failed` or `sciencedirect_robot_check`.
- Ask the user to solve it in Chrome.
- Then continue once from the same now-open page.
- If the same challenge immediately reappears, mark `do_not_auto_retry` and move on.

If shell `Invoke-WebRequest` or `curl` returns 403 but the PDF opens in Chrome:

- Use `browser_pdf_downloader.mjs`; this is the normal institutional-access case.

If a page shows publisher bot verification, CAPTCHA, Cloudflare, QR login, SMS/OTP, or another security challenge:

- Do not ask for or accept institutional credentials in chat. Publisher API keys follow the separate save-on-receipt rule in `../SKILL.md`.
- Pause and ask the user to complete the verification in Chrome.
- Record `publisher_verification_waiting_user` in `publisher_verification.tsv`, or `sciencedirect_robot_check` for ScienceDirect.
- Continue only after the user says the browser step is complete.

If a page shows institutional SSO, CAS, CARSI/Shibboleth, OpenAthens, SAML, federation/WAYF/机构选择, database login, or IP-login options:

- Do not ask for or accept institutional credentials in chat. Publisher API keys follow the separate save-on-receipt rule in `../SKILL.md`.
- If the user has explicitly authorized it and the browser has already filled credentials, click the visible login/confirm button once.
- Otherwise pause and ask the user to complete the login in the browser.
- Record `carsi_waiting_user` or `carsi_resolved_retry_needed` in `carsi_retry.tsv` as appropriate.

If the aggregation entry shows no full-text link:

- Try the publisher's own `Institutional login` / `机构登录` / CARSI/Shibboleth/OpenAthens route and select the user's institution when authorized.
- Try the DOI on the publisher page once an institutional session exists.
- Check open-access copies only from legitimate sources.
- Record `no_authorized_pdf_found` rather than seeking unauthorized mirrors.

If a page opens as `about:blank`:

- Treat it as a URL-fragment/encoding problem first, especially when the original URL contains `#` or `#!`.
- Reopen through `scripts/cdp_open_url.mjs --url "<full URL>" --wait`.
- Do not paste fragment-heavy URLs unquoted into shell commands or manually concatenate them into `/new?url=...` without URL encoding.

If `curl` is unavailable:

- Use PowerShell `Invoke-WebRequest` for simple proxy checks.
- Prefer the bundled Node.js helper scripts for CDP proxy actions because Node's `URLSearchParams` preserves nested URL fragments correctly.

If the session expires:

- Ask the user to re-authenticate through their institution/library route in the same browser, then reopen the publisher/database entry.

## To Confirm With The User on First Run

These items depend on the user's live institution/library session and should be confirmed once per deployment or institution profile:

1. The exact institutional login, federation, proxy, WebVPN, or database hosts that appear in the address bar.
2. The base URL / link pattern of the library aggregation or database entries the user actually uses.
3. Whether a federation/WAYF/机构选择, IP-login, or database-login step appears, and whether the user authorizes selecting the unambiguous institution/login option.

# Institutional Browser Workflow

## Contents

- [Recommended Web Access Workflow](#recommended-web-access-workflow)
- [Publisher Verification and ScienceDirect](#publisher-verification-and-sciencedirect)
- [Institutional Authentication Handoff and Retry](#institutional-authentication-handoff-and-retry)
- [Download PDF From Browser Context](#download-pdf-from-browser-context)


Load this reference only when lawful open-access sources are unavailable and the selected route
requires the user's institution-authorized browser session. It covers Web of Science discovery,
publisher verification, institutional authentication handoff, and PDF transfer from the browser
context.

## Recommended Web Access Workflow

Use this section only after legitimate OA sources are unavailable: directly for English publishers outside Elsevier/Springer Nature/IEEE, or after the user explicitly accepts Web Access fallback when both a supported publisher API and the OA fallback failed. Start from Web of Science or the configured library portal and reuse the user's authenticated browser session.

Before using the library route, check for legitimate open-access availability when the article metadata suggests OA or the user provides an OA/open journal paper. Use PMC, publisher OA links, arXiv, DOI landing pages with clear open PDF access, or a known lawful PDF URL. If an OA PDF is available, download and verify it directly, mark `open_access_downloaded`, and record the OA source in the manifest. Do not require institutional login for an article that is already openly available.

Important distinction: `--topic` is a Web of Science topic search, not an exact-title resolver. For a known exact title, especially conference/arXiv papers without DOI, prefer `--title "<exact title>" --open-access` or `--pdf-url` when the legitimate PDF URL is known. In testing, `--topic "Attention Is All You Need"` matched an unrelated HBR article first, while `--title "Attention Is All You Need" --open-access` correctly downloaded arXiv `1706.03762v7`.

Web of Science hosts to recognize: `webofscience.clarivate.cn`, `www.webofscience.com`, `*.webofknowledge.com`, `*.clarivate.com`. Note: WoS renders records inside **shadow DOM with a virtualized list** — when scraping manually you must pierce shadow roots and scroll to load more rows (the batch script already does this).

1. **Authenticate once**: open Web of Science via the library aggregation / institutional entry. If Web of Science or another database shows authentication choices such as institutional login, Shibboleth/OpenAthens/CARSI, CAS/SSO, or IP login, use the route the user normally uses. If credentials, QR, CAPTCHA, SMS/OTP, or unclear consent appears, follow **Institutional Authentication Handoff** below.
2. Confirm you are on the authenticated Web of Science search page (institutional name visible, search box present).
3. Search the paper by **DOI** when available, otherwise by **exact title**:
   - Set the search field to `DOI` or `Title`, paste the value, run the search.
   - Read the results page with `/eval` and pick the record that matches title + year + authors.
4. Open the matching record and read it with `/eval`.
5. Click the full-text route, in this order of preference:
   - `Free Full Text` / `Open Access` if present
   - library resolver links: `Find it at`, `SFX`, `OpenURL`, `Full Text Links`, `查看全文`, `Full Text available via`, database/provider names such as Ovid
   - publisher full-text link: `View Full Text`, the publisher name, or `View PDF`
   - The full-text link should inherit the institutional session, so the publisher often grants access without a second login. If a second institutional handoff appears, complete it once.
6. On the publisher page, find the PDF link (`PDF`, `View PDF`, `Download PDF`, `pdfft`, `/doi/pdf/`) and save it with `scripts/browser_pdf_downloader.mjs`.
7. If the full-text resolver opens readable HTML full text but no valid PDF is exposed, save the HTML/text, mark `full_text_html_available`, and tell the user plainly: "已获取 HTML 全文，但当前授权路径没有可下载 PDF." Do not mislabel an HTML page as a PDF; if a PDF probe returns HTML, move it to diagnostics and explain that no valid PDF was downloaded.
8. If the resolver/provider explicitly says the institution has no entitlement, mark `library_no_permission` and tell the user: "当前图书馆资源没有该文献全文权限." Do not hide this behind `failed_after_retry`.
9. **Do not download Supporting Information by default.** Only fetch SI if the user explicitly asked; otherwise just note whether SI exists (see `delivery-verification-and-failures.md`).
10. Record the route taken (OA source or WoS → SFX/OpenURL/full-text provider → publisher/database) in the manifest.

If Web of Science returns no record, or the record has no accessible full-text link, mark the paper `no_full_text_link` and tell the user. If the library route is found but denies entitlement, mark `library_no_permission`. Do not silently fall back to direct publisher navigation as if it were the same authorized route.

## Publisher Verification and ScienceDirect

ScienceDirect and some publisher platforms may show "Are you a robot?", CAPTCHA, Cloudflare, bot verification, or similar checks after repeated direct DOI navigation or automated tab opening. These pages are security and anti-automation challenges, not ordinary login confirmations.

Reduce the chance of triggering them by using a conservative access pattern:

1. Prefer the library aggregation / CARSI entry before direct `doi.org -> publisher` navigation.
2. Process ScienceDirect and other sensitive publishers one article at a time.
3. Keep a visible audit trail in the manifest; do not open many publisher tabs in parallel.
4. Wait for each page to settle before looking for `Download PDF`, `View PDF`, or `PDF`.
5. Reuse the same tab after the user completes a verification step instead of opening repeated new tabs.
6. Avoid retry loops. Use one attempt by default and no more than two attempts on the same tab before handing the page to the user.

When a publisher verification page appears:

1. First, **attempt automatic verification** via the built-in anti-bot module (`scripts/lib/anti-bot.mjs`). The module tries: simple click challenges, ScienceDirect robot check, Cloudflare Turnstile, slider CAPTCHA (including CNKI Geetest-style), and reCAPTCHA checkbox.
2. If auto-verification succeeds, continue the download from the resolved page.
3. If auto-verification fails:
   a. Stop automated actions on that tab.
   b. Record the paper with status `verification_auto_failed`. Use `sciencedirect_robot_check` only when no automatic interaction was possible.
   c. Tell the user which paper and tab need manual attention.
   d. After the user says the verification is complete, continue from the same tab and try the visible article/PDF route once.
   e. If verification immediately reappears, mark `do_not_auto_retry` and move on.

Create or update `publisher_verification.tsv` when publisher checks interrupt a batch. Use this header:

```text
id	project	title	doi	year	venue	publisher	status	source_url	current_url	next_action	notes
```

Suggested `next_action` values:

```text
user_complete_publisher_verification
retry_same_tab_after_user_confirms
try_aggregation_entry_route
try_authorized_oa_route
mark_do_not_auto_retry
```

## Institutional Authentication Handoff and Retry

Publishers and databases routed through CAS/SSO, CARSI/Shibboleth, OpenAthens, EZproxy, WebVPN, or IP authorization may redirect to an institutional login or database login page for the first authenticated access. This is expected and is not a reason to ask for the user's password.

When a page reaches an institutional login page, federation/WAYF selector, database login page, or IP-login prompt:

1. Stop automated actions on that tab.
2. Record the paper in `carsi_retry.tsv` with status `carsi_waiting_user`.
3. Tell the user exactly which tab/page needs attention, for example: "This page is at your institution/database login. If the browser has already filled credentials, I can click the visible login/confirm button once with your authorization; otherwise please complete it in the browser." If a federation/WAYF page asks which institution to use, ask the user to pick their institution, or do it only when the choice is unambiguous and credential-free and the user authorized it.
4. Do not read, store, or request the password, QR result, OTP, SMS code, CAPTCHA, cookie, or local/session storage.
5. If the user explicitly authorizes clicking because credentials are already filled, click only the visible login/confirm/continue button once. Do not type into fields or inspect hidden credential values. For credential-free options such as "IP login", click only when the user authorizes that route or has just completed it manually.
6. If QR login, SMS/OTP, CAPTCHA, Cloudflare, or publisher bot verification appears, stop and let the user complete it manually.
7. After the login/confirm step completes, refresh or continue from the same tab.
8. Re-detect whether the page is now a publisher article page, a PDF viewer, or another institutional handoff.
9. If resolved, download and verify the PDF/SI, then update the manifest status to `downloaded` or `downloaded_with_si`.
10. If it loops back to the same institutional/database login after a completed user login, record `failed_after_retry` with the observed reason and move on.

### Safe Institutional Auto-Confirm

The agent may click a saved-login confirmation button only when all conditions are true:

```text
1. The page is on an expected institutional, library, federation, or database domain for the user's configured route.
2. The user has explicitly authorized this action in the current conversation, for example: "可以点这个机构登录确认按钮".
3. The visible action is clearly a login/confirm/continue button, such as 登录, 登 录, 确认登录, 继续登录, Continue, Proceed, or Sign in.
4. There is no visible QR-only login, SMS/OTP field, push-approval prompt, password reset prompt, consent-to-share-new-data prompt, or account/security warning. (Slider CAPTCHAs and simple robot checks are now auto-attemptable — see Boundaries in `../SKILL.md`.)
5. The agent does not read, reveal, copy, store, type, or modify credentials.
```

A federation/WAYF/机构选择 page carries no credentials and may be selected when the institution is unambiguous and the user has authorized it. If any condition is unclear, pause and ask the user to handle that tab. Do not repeatedly click login; one click is enough to test whether the saved-login state works.

Create or update `carsi_retry.tsv` whenever institutional authentication blocks a batch. Use this header:

```text
id	project	title	doi	year	venue	publisher	failure_stage	status	source_url	current_url	next_action	notes
```

Suggested `next_action` values:

```text
user_complete_institution_login_in_chrome
select_institution_in_federation_wayf
retry_same_tab_after_user_confirms
repair_url_by_doi
try_aggregation_entry_route
mark_no_authorized_pdf
```

For a CARSI retry batch, process one or a few tabs at a time. Do not open many login tabs in parallel; it can confuse the user's session and increase publisher or SSO risk.

## Download PDF From Browser Context

Use the bundled script when a PDF URL opens in Chrome but direct shell download returns `403`, `401`, Cloudflare HTML, or a login page.

```powershell
$node = "$env:LOCALAPPDATA\OpenAI\Codex\bin\node.exe"
& $node "$env:USERPROFILE\.agents\skills\nature-downloader\scripts\browser_pdf_downloader.mjs" `
  --url "https://www.sciencedirect.com/science/article/pii/SXXXXXXXXXXXXXXXX/pdfft" `
  --out "D:\path\paper.pdf"
```

The script:

- Opens the URL in the user's controlled Chrome session unless `--target` is provided.
- Runs `fetch(location.href, { credentials: "include" })` inside the page.
- Transfers bytes in chunks through the local CDP proxy.
- Writes the binary file to disk.
- Verifies the `%PDF` signature by default.

Useful options:

```text
--url <url>          PDF URL to open and save
--target <targetId>  Existing Chrome target/tab id to use
--out <path>         Output PDF path
--proxy <url>        CDP proxy URL, default http://127.0.0.1:3456
--close              Close the tab after download if the script opened it
--allow-non-pdf      Save even when content does not start with %PDF
```

pdfplumber>=0.11.0
pypdf>=4.0.0
PyYAML>=6.0.0
jsonschema>=4.0.0

#!/usr/bin/env node
// Institution-neutral literature downloader.
//
// Runs the whole chain inside Node + the web-access CDP proxy, so large data
// (search DOMs, PDF bytes) never enters the agent's context. Only compact
// per-paper status is printed. This is the token-efficient fast path.
//
// Usage:
//   Every download requires exactly one of --si / --no-si.
//   node batch_download.mjs --topic "<query>" --count 10 --out <dir> --no-si
//   node batch_download.mjs --dois 10.x/a,10.y/b --out <dir> --si
//   node batch_download.mjs --title "<exact title>" --out <dir> --no-si
//   node batch_download.mjs --pdf-url "https://..." --title "<title>" --out <dir> --no-si
//   options: [--proxy http://127.0.0.1:3456] [--debug] [--legacy-status]
//
// Boundaries: uses only the user's already-authenticated browser session.
// Stops at institutional SSO / CARSI pages and never handles credentials. For visible
// verification widgets it makes a bounded attempt before reporting a handoff.
// Main PDF only by default; --si also fetches supplements.

import fs from "node:fs";
import crypto from "node:crypto";
import path from "node:path";
import { pathToFileURL } from "node:url";
import {
  DEFAULT_PROXY,
  healthCheck,
  evalJs,
  newTab,
  navigate,
  closeTab,
  listTargets,
  click,
  scroll,
  waitForComplete,
  proxyGet,
} from "./lib/cdp-utils.mjs";
import { classifyWall, STATUS, isSuccess } from "./lib/status-codes.mjs";
import { fetchToFile, fetchAnyToFile } from "./lib/pdf-utils.mjs";
import { handleVerification } from "./lib/anti-bot.mjs";
import {
  DEFAULT_DISCOVERY_URL,
  discoveryUrlFromConfig,
  loadSchoolConfig,
  schoolSummary,
} from "./lib/school-config.mjs";
import { findArxivByTitle } from "./lib/open-access.mjs";
import {
  articleBundleDirectory,
  downloadWosSupportingInformation,
  exactArticleTitleMatch,
  safeArticleTitle,
  shouldUseCleanWosBundle,
  wosSearchQuery,
} from "./lib/wos-supporting-information.mjs";
import {
  DEFAULT_CNKI_URL,
  createCnkiTransport,
  downloadCnkiDirectUrl,
  downloadCnkiTitle,
  looksChinese,
} from "./lib/cnki.mjs";
import {
  classifyPublisher,
  chooseDirectUrlRoute,
  hasUsablePublisherCredentials,
  parseSiChoice,
  chooseRoute,
  isChineseLiterature,
} from "./lib/routing.mjs";
import { fetchCrossrefByDoi, findCrossrefByTitle } from "./lib/metadata.mjs";
import { downloadOpenAccessArticle } from "./lib/open-access-provider.mjs";
import { downloadPublisherArticle } from "./lib/publisher-providers.mjs";
import { providerCredentials } from "./lib/credentials.mjs";
import { loadSettings } from "./lib/settings.mjs";
import { saveFullTextResponse } from "./lib/direct-download.mjs";
import { writeManifest } from "./lib/manifest.mjs";

// Core Collection only: journal articles that carry DOIs (avoids Derwent/patent records).
const WOS = DEFAULT_DISCOVERY_URL;

export function parseArgs(argv) {
  const a = { out: ".", si: false, noSi: false, proxy: DEFAULT_PROXY, debug: false, legacyStatus: false };
  for (let i = 2; i < argv.length; i++) {
    const k = argv[i];
    if (k === "--topic") a.topic = argv[++i];
    else if (k === "--title") a.title = argv[++i];
    else if (k === "--pdf-url") a.pdfUrl = argv[++i];
    else if (k === "--open-access") a.openAccess = true;
    else if (k === "--cnki-url") a.cnkiUrl = argv[++i];
    else if (k === "--cnki-format") a.cnkiFormat = argv[++i];
    else if (k === "--dois") a.dois = argv[++i].split(",").map((s) => s.trim()).filter(Boolean);
    else if (k === "--count") a.count = Number(argv[++i]);
    else if (k === "--out") a.out = argv[++i];
    else if (k === "--si") a.si = true;
    else if (k === "--no-si") a.noSi = true;
    else if (k === "--language") a.language = argv[++i];
    else if (k === "--route") a.route = argv[++i];
    else if (k === "--unpaywall-email") a.unpaywallEmail = argv[++i];
    else if (k === "--source-url") a.sourceUrl = argv[++i];
    else if (k === "--api-fallback-web") a.apiFallbackWeb = true;
    else if (k === "--no-api-fallback-web") a.apiFallbackWeb = false;
    else if (k === "--api-fallback-web-for") a.apiFallbackWebFor = argv[++i].split(",").map((s) => s.trim()).filter(Boolean);
    else if (k === "--no-api-fallback-web-for") a.noApiFallbackWebFor = argv[++i].split(",").map((s) => s.trim()).filter(Boolean);
    else if (k === "--proxy") a.proxy = argv[++i].replace(/\/$/, "");
    else if (k === "--debug") a.debug = true;
    else if (k === "--legacy-status") a.legacyStatus = true;
    else throw new Error("unknown arg " + k);
  }
  const primaryModes = [a.topic, a.pdfUrl, a.dois?.length].filter(Boolean).length;
  if (primaryModes > 1 || (a.title && a.dois?.length)) {
    throw new Error("--topic, --pdf-url, and --dois are mutually exclusive; --title may accompany --topic or --pdf-url");
  }
  if (a.cnkiFormat && !["pdf", "any"].includes(a.cnkiFormat)) {
    throw new Error("--cnki-format must be pdf or any");
  }
  if (a.route && !["cnki", "open_access", "elsevier", "springer_nature", "ieee", "web_access"].includes(a.route)) {
    throw new Error("--route must be cnki, open_access, elsevier, springer_nature, ieee, or web_access");
  }
  parseSiChoice(a);
  if (!a.topic && !a.title && !a.pdfUrl && !a.dois?.length) throw new Error("one of --topic, --title, --pdf-url, or --dois is required");
  if (a.topic && a.count === undefined) a.count = 10;
  return a;
}

export function selectBatchDois(dois = [], count) {
  const unique = [...new Set(dois)];
  return count === undefined ? unique : unique.slice(0, count);
}

function cnkiUrlFromConfig(config, argUrl) {
  return (
    argUrl ||
    config?.discovery?.cnki_url ||
    config?.discovery?.cnki ||
    DEFAULT_CNKI_URL
  );
}

async function handleWosAuthPreference(proxy, target) {
  const info = await proxyGet(proxy, "/info", { target }, 8000).catch(() => ({}));
  const marker = `${info.url || ""} ${info.title || ""}`;
  if (!/AUTH_PREFERENCE_ERROR|身份验证首选项|Authentication Preference/i.test(marker)) {
    return false;
  }
  const clicked = await evalJs(
    proxy,
    target,
    `(()=>{const r=document.querySelector('#radio-shibboleth,input[value="shibboleth"]');if(r)r.click();const b=[...document.querySelectorAll('button,a,input[type=button],input[type=submit]')].find(e=>/(继续|Continue)/i.test(e.innerText||e.value||''));if(b)b.click();return !!b;})()`
  ).catch(() => false);
  if (clicked) {
    await new Promise((r) => setTimeout(r, 3000));
    await waitForComplete(proxy, target);
  }
  return Boolean(clicked);
}

// --- WoS: search a topic, return the first N full-record URLs ---
async function wosRecordUrls(proxy, topic, count, debug, discoveryUrl = WOS) {
  const tabs = await listTargets(proxy);
  let target = (tabs.find((t) => /webofscience\./i.test(t.url || "")) || {}).targetId;
  if (target) await navigate(proxy, target, discoveryUrl);
  else target = (await newTab(proxy, discoveryUrl)).targetId;
  await waitForComplete(proxy, target);
  await handleWosAuthPreference(proxy, target);
  await new Promise((r) => setTimeout(r, 1500));
  await evalJs(
    proxy,
    target,
    `(()=>{const a=document.querySelector('#onetrust-accept-btn-handler');if(a)a.click();return 1;})()`
  ).catch(() => {});
  await evalJs(
    proxy,
    target,
    `(()=>{const i=document.querySelector('#search-option-0');if(!i)return 0;const s=Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype,'value').set;s.call(i,${JSON.stringify(topic)});i.dispatchEvent(new Event('input',{bubbles:true}));return 1;})()`
  );
  await click(proxy, target, 'button[data-ta="run-search"]');
  // WoS renders records in shadow DOM + a virtualized list, so we walk shadow roots
  // and scroll to load more rows until we have enough record links.
  const collect = `(()=>{const out=new Set();(function w(r){r.querySelectorAll('*').forEach(e=>{if(e.shadowRoot)w(e.shadowRoot);if(e.tagName==='A'&&/\\/full-record\\//.test(e.href||''))out.add(e.href);});})(document);return JSON.stringify([...out]);})()`;
  let urls = [];
  let lastInfo = null;
  for (let i = 0; i < 25; i++) {
    await new Promise((r) => setTimeout(r, 1000));
    const inf = await proxyGet(proxy, "/info", { target }, 8000).catch(() => ({}));
    lastInfo = inf;
    if (!/\/summary\//.test(inf.url || "")) continue;
    const found = JSON.parse((await evalJs(proxy, target, collect)) || "[]");
    if (found.length > urls.length) urls = found;
    if (urls.length >= count * 2) break;
    await scroll(proxy, target, "bottom");
  }
  if (debug && urls.length === 0) {
    const html = await evalJs(
      proxy,
      target,
      `document.documentElement.outerHTML.slice(0,5000)`
    ).catch(() => "");
    process.stderr.write(`[debug][wos] no records found. url=${lastInfo?.url||'?'} title=${lastInfo?.title||'?'}\n`);
    process.stderr.write(`[debug][wos] html snippet: ${html.slice(0, 500)}\n`);
  }
  return { target, urls: urls.slice(0, count * 3) };
}

// --- From a WoS full-record page, get the article DOI ---
async function doiFromRecord(proxy, target, recordUrl) {
  await navigate(proxy, target, recordUrl);
  await waitForComplete(proxy, target);
  await new Promise((r) => setTimeout(r, 1200));
  const doi = await evalJs(
    proxy,
    target,
    `(()=>{let h='';(function w(r){r.querySelectorAll('*').forEach(e=>{if(e.shadowRoot)w(e.shadowRoot);if(!h&&e.tagName==='A'&&/doi\\.org\\/10\\./.test(e.href||''))h=e.href;});})(document);if(h)return (h.match(/10\\.\\d{4,9}\\/[^\\s"?]+/)||[])[0];const m=(document.body.innerText||'').match(/10\\.\\d{4,9}\\/[^\\s"]+/);return m?m[0]:'';})()`
  );
  return (doi || "").replace(/[.,;]+$/, "");
}

async function captureBrowserFullTextHtml(proxy, tab, outDir, doi, fallbackTitle = "") {
  const raw = await evalJs(
    proxy,
    tab,
    `(()=>{const root=document.querySelector('article,.article-body,.article__body,.c-article-body,#body,#fulltext');if(!root)return '';const text=(root.innerText||'').trim();if(text.length<500)return '';const title=document.querySelector('meta[name=citation_title]')?.content||document.querySelector('h1')?.innerText||${JSON.stringify(fallbackTitle)};return JSON.stringify({title,html:'<!doctype html><meta charset="utf-8"><title>'+String(title).replace(/[<&]/g,'')+'</title>'+root.outerHTML});})()`
  ).catch(() => "");
  if (!raw) return null;
  const value = JSON.parse(raw);
  const safe = safeArticleTitle(value.title || fallbackTitle || doi);
  const file = path.join(outDir, "FullText", `${safe}.html`);
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.writeFileSync(file, value.html, "utf8");
  return { file, bytes: Buffer.byteLength(value.html), format: "html", status: STATUS.FULL_TEXT_HTML_AVAILABLE };
}

// --- Download main PDF (and optionally SI) for a DOI via the authenticated browser ---
async function downloadDoi(proxy, doi, outDir, wantSi, debug, { cleanBundle = false, bundleTitle = "" } = {}) {
  const tab = (await newTab(proxy, "https://doi.org/" + doi)).targetId;
  try {
    const info = await waitForComplete(proxy, tab);
    const wall = classifyWall(info.url || "", info.title || "");
    if (wall) {
      // Attempt automatic verification before giving up.
      if (debug) process.stderr.write(`[debug][doi] wall detected: ${wall.status} "${wall.reason}" — attempting auto-verification...\n`);
      const verifyResult = await handleVerification(proxy, tab, wall, { debug, maxAttempts: 1 });
      if (verifyResult.passed) {
        if (debug) process.stderr.write(`[debug][doi] auto-verification passed (${verifyResult.method}), retrying...\n`);
        // Verification passed — re-read the page state and continue the download loop.
        const newInfo = await waitForComplete(proxy, tab);
        const reWall = classifyWall(newInfo.url || "", newInfo.title || "");
        if (!reWall) {
          // Wall is gone, fall through to PDF candidate extraction below.
          // We need to re-read meta candidates, so jump back to the metadata polling loop.
          info.url = newInfo.url;
          info.title = newInfo.title;
          // Clear the title so waitForComplete re-reads it; fall through
        } else {
          // Still blocked after auto-verification — hand off to user.
          return { doi, status: STATUS.VERIFICATION_AUTO_FAILED, url: info.url, reason: `auto-verify failed (${verifyResult.method || 'unknown'}), still: ${reWall.reason}` };
        }
      } else {
        return {
          doi,
          status: verifyResult.attempted ? STATUS.VERIFICATION_AUTO_FAILED : wall.status,
          url: info.url,
          reason: verifyResult.attempted ? `automatic verification did not resolve: ${wall.reason}` : wall.reason,
        };
      }
    }
    // poll: publisher landings often JS-redirect (e.g. linkinghub -> sciencedirect)
    // and inject citation_pdf_url late; re-read a few times before giving up.
    let meta = {};
    for (let i = 0; i < 4; i++) {
      await new Promise((r) => setTimeout(r, 1200));
      meta = JSON.parse(
        (await evalJs(
          proxy,
          tab,
          `(()=>{
        const m=document.querySelector('meta[name=citation_pdf_url]');
        const cand=[]; if(m&&m.content)cand.push(m.content);
        document.querySelectorAll('a').forEach(a=>{const h=a.href||'';if(/\\/pdf|pdfdirect|pdfft|\\.pdf(\\?|$)|\\/doi\\/epdf/i.test(h))cand.push(h);});
        const articleTitle=document.querySelector('meta[name=citation_title]')?.content||document.querySelector('meta[property="og:title"]')?.content||document.querySelector('h1')?.innerText||'';
        return JSON.stringify({cand:[...new Set(cand)].slice(0,6),title:document.title||'',articleTitle,url:location.href,body:(document.body.innerText||'').slice(0,160)});
      })()`
        )) || "{}"
      );
      const w = classifyWall(meta.url || "", meta.title || "", meta.body || "");
      if (w) {
        // Attempt auto-verification before giving up.
        if (debug) process.stderr.write(`[debug][doi] meta-stage wall: ${w.status} "${w.reason}" — attempting auto-verification...\n`);
        const vr = await handleVerification(proxy, tab, w, { debug, maxAttempts: 1 });
        if (vr.passed) {
          if (debug) process.stderr.write(`[debug][doi] auto-verification passed (${vr.method}), continuing...\n`);
          // Wall is gone after verification, re-poll meta
          await new Promise((r) => setTimeout(r, 1500));
          continue;
        }
        return {
          doi,
          status: vr.attempted ? STATUS.VERIFICATION_AUTO_FAILED : w.status,
          url: meta.url,
          reason: vr.attempted ? `automatic verification did not resolve: ${w.reason}` : w.reason,
        };
      }
      if (meta.cand && meta.cand.length) break;
    }
    // Publisher quirk: Wiley's citation_pdf_url/epdf opens a viewer, not raw bytes.
    // The pdfdirect?download=true endpoint returns the actual PDF in the same session.
    if (/wiley\.com/i.test(meta.url || "") || (meta.cand || []).some((c) => /wiley\.com/i.test(c))) {
      meta.cand = [
        `https://onlinelibrary.wiley.com/doi/pdfdirect/${doi}?download=true`,
        ...(meta.cand || []),
      ];
    }
    // Distinguish WoS-stage "no record" from publisher-stage "no PDF":
    // if we got here, WoS found the record (we have a doi.org redirect to a
    // publisher page), so "no PDF candidates" means no_authorized_pdf_found.
    if (!meta.cand || !meta.cand.length) {
      const html = await captureBrowserFullTextHtml(proxy, tab, outDir, doi, bundleTitle || meta.articleTitle || "");
      if (html) return { doi, ...html, via: meta.url, provider: "web_access", accessMode: "institution_browser" };
      if (debug) {
        process.stderr.write(
          `[debug][doi] ${doi} no PDF candidates. url=${meta.url||'?'} title=${meta.title||'?'}\n`
        );
      }
      return { doi, status: STATUS.NO_AUTHORIZED_PDF_FOUND, url: meta.url };
    }

    if (cleanBundle && bundleTitle && (!meta.articleTitle || !exactArticleTitleMatch(bundleTitle, meta.articleTitle))) {
      return {
        doi,
        title: meta.articleTitle || "",
        status: STATUS.DO_NOT_AUTO_RETRY,
        url: meta.url,
        reason: meta.articleTitle
          ? `WoS title mismatch: expected "${bundleTitle}"`
          : `WoS title could not be verified: expected "${bundleTitle}"`,
      };
    }

    const safe = doi.replace(/[\/:*?"<>|]/g, "_");
    const articleTitle = bundleTitle || meta.articleTitle || doi;
    const bundleDir = cleanBundle ? articleBundleDirectory(outDir, articleTitle) : "";
    const pdfPath = cleanBundle
      ? path.join(bundleDir, `${safeArticleTitle(articleTitle)}.pdf`)
      : path.join(outDir, "PDFs", safe + ".pdf");
    for (const pdfUrl of meta.cand) {
      const got = await fetchToFile(proxy, tab, pdfUrl, pdfPath);
      if (got.ok) {
        const res = {
          doi,
          ...(cleanBundle ? { title: articleTitle } : {}),
          status: STATUS.DOWNLOADED,
          file: got.file,
          bytes: got.bytes,
          via: meta.url,
        };
        if (wantSi) {
          const si = cleanBundle
            ? await downloadWosSupportingInformation({
                proxy,
                tab,
                landingUrl: meta.url,
                bundleDir,
                reservedFilenames: [path.basename(pdfPath)],
              })
            : await downloadSi(proxy, tab, meta.url, doi, outDir);
          res.si = si;
          if ((si?.downloaded || si?.count || 0) > 0) res.status = STATUS.DOWNLOADED_WITH_SI;
        }
        return res;
      }
    }
    return { doi, status: STATUS.PDF_FETCH_FAILED, url: meta.url };
  } finally {
    await closeTab(proxy, tab);
  }
}

async function downloadDirectUrl(pdfUrl, outDir, title = "") {
  const response = await fetch(pdfUrl, { headers: { Accept: "application/pdf, text/html, application/xml" }, signal: AbortSignal.timeout(60000) });
  const saved = await saveFullTextResponse(response, { outDir, title, source: pdfUrl });
  if (!saved.ok) return { title, status: STATUS.PDF_FETCH_FAILED, url: pdfUrl, err: saved.reason || `HTTP ${saved.httpStatus}` };
  return {
    title,
    status: saved.format === "pdf"
      ? STATUS.OPEN_ACCESS_DOWNLOADED
      : saved.format === "html"
        ? STATUS.FULL_TEXT_HTML_AVAILABLE
        : STATUS.NATIVE_FULLTEXT_DOWNLOADED,
    provider: "direct_url",
    accessMode: "open_access",
    ...saved,
  };
}

async function downloadSi(proxy, tab, landingUrl, doi, outDir) {
  // Best-effort: scan landing page for supplement links, download each.
  await navigate(proxy, tab, landingUrl);
  await waitForComplete(proxy, tab);
  await new Promise((r) => setTimeout(r, 800));
  const links = JSON.parse(
    (await evalJs(
      proxy,
      tab,
      `JSON.stringify([...new Set(Array.from(document.querySelectorAll('a')).map(a=>a.href).filter(h=>/downloadSupplement|\\/suppl|supplementary|mmc\\d|_si_/i.test(h)))].slice(0,30))`
    )) || "[]"
  );
  const safe = doi.replace(/[\/:*?"<>|]/g, "_");
  let n = 0;
  for (const u of links) {
    const name = (u.split("file=")[1] || u.split("/").pop() || "si" + ++n)
      .replace(/[\/:*?"<>|]/g, "_")
      .slice(0, 80);
    const got = await fetchAnyToFile(
      proxy,
      tab,
      u,
      path.join(outDir, "SupportingInformation", safe + "__" + name)
    );
    if (got.ok) n++;
  }
  return { count: n, found: links.length };
}

async function downloadSiForDoi(proxy, doi, outDir) {
  const tab = (await newTab(proxy, `https://doi.org/${doi}`)).targetId;
  try {
    const info = await waitForComplete(proxy, tab);
    const wall = classifyWall(info.url || "", info.title || "");
    if (wall) return { status: wall.status, count: 0, found: 0, reason: wall.reason };
    const result = await downloadSi(proxy, tab, info.url, doi, outDir);
    return { status: result.count > 0 ? "downloaded" : "not_found", ...result };
  } finally {
    await closeTab(proxy, tab);
  }
}

function decorateResult(result, article, route, wantSi) {
  const integrity = result.file && fs.existsSync(result.file)
    ? {
        sha256: result.sha256 || crypto.createHash("sha256").update(fs.readFileSync(result.file)).digest("hex"),
        format: result.format || path.extname(result.file).replace(/^\./, "").toLowerCase(),
      }
    : {};
  return {
    doi: article.doi || result.doi || "",
    title: article.title || result.title || "",
    language: article.language || "",
    publisher: article.publisher || "",
    route: route.provider,
    route_reason: route.reason,
    si_requested: wantSi,
    ...result,
    ...integrity,
  };
}

async function attachSupportingInformation(result, doi, args, context) {
  if (!args.si || !isSuccess(result.status)) return result;
  try {
    await context.ensureBrowser();
    result.si = await downloadSiForDoi(args.proxy, doi, args.out);
  } catch (error) {
    result.si = { status: "fetch_failed", count: 0, reason: String(error?.message || error).slice(0, 160) };
  }
  if ((result.si?.count || 0) > 0) result.status = STATUS.DOWNLOADED_WITH_SI;
  return result;
}

async function runWebAccess(doi, article, args, context) {
  await context.ensureBrowser();
  return downloadDoi(args.proxy, doi, args.out, args.si, args.debug, {
    cleanBundle: shouldUseCleanWosBundle(args),
    bundleTitle: args.title || article.title || "",
  });
}

async function downloadArxivTitle(title, args) {
  const hit = await findArxivByTitle(title).catch(() => null);
  if (!hit?.pdfUrl) return null;
  const downloaded = await downloadDirectUrl(hit.pdfUrl, args.out, hit.title).catch((error) => ({ status: STATUS.FAILED_AFTER_RETRY, err: String(error).slice(0, 120) }));
  return { ...downloaded, title: hit.title, arxiv: hit.id, route: "open_access", route_reason: "arxiv_exact_title", si_requested: args.si, ...(args.si ? { si: { status: "not_found" } } : {}) };
}

async function downloadCnkiByTitle(args, context) {
  if (!args.title) {
    return {
      title: "",
      status: STATUS.METADATA_AMBIGUOUS,
      reason: "CNKI title search requires --title when --pdf-url is not a CNKI URL",
      source: "cnki",
    };
  }
  await context.ensureBrowser();
  process.stderr.write(`[cnki] searching Chinese title: ${args.title}\n`);
  return downloadCnkiTitle(args.proxy, args.title, args.out, {
    cnkiUrl: context.cnkiUrl,
    format: args.cnkiFormat || "any",
    debug: args.debug,
    wantSi: args.si,
    transport: context.cnkiTransport,
  }).catch((error) => ({
    title: args.title,
    status: STATUS.FAILED_AFTER_RETRY,
    err: String(error).slice(0, 120),
    source: "cnki",
  }));
}

function attemptSummary(result, provider) {
  if (!result) return null;
  return {
    provider,
    status: result.status,
    ...(result.httpStatus ? { http_status: result.httpStatus } : {}),
  };
}

async function routeDoiDownload(doi, args, context) {
  let article;
  try {
    article = await fetchCrossrefByDoi(doi, { mailto: context.contactEmail });
  } catch (error) {
    article = { doi, title: args.title || doi, language: args.language || "", metadata_error: String(error?.message || error).slice(0, 160) };
  }
  if (args.title && context.singleDoi) article.title = args.title;
  if (args.language) article.language = args.language;
  if (args.sourceUrl) article.sourceUrl = args.sourceUrl;
  if (args.route) article.routeOverride = args.route;

  if (isChineseLiterature(article) || args.route === "cnki") {
    await context.ensureBrowser();
    const route = { provider: "cnki", reason: args.route ? "explicit_override" : "chinese_literature" };
    const result = await downloadCnkiTitle(args.proxy, article.title || doi, args.out, {
      cnkiUrl: context.cnkiUrl,
      format: args.cnkiFormat || "any",
      debug: args.debug,
      wantSi: args.si,
      transport: context.cnkiTransport,
    });
    return decorateResult(result, article, route, args.si);
  }

  const publisherProvider = classifyPublisher(article);
  const publisherCredentials = publisherProvider === "other" ? null : providerCredentials(publisherProvider);
  const initialRoute = chooseRoute({
    ...article,
    routeOverride: args.route,
    hasPublisherCredentials: hasUsablePublisherCredentials(publisherProvider, publisherCredentials),
  });
  const useApiFirst = !args.route && initialRoute.reason === "publisher_api_credentials_available";
  let apiResult = null;
  let oaResult = null;

  if (useApiFirst) {
    apiResult = await downloadPublisherArticle(article, {
      provider: publisherProvider,
      credentials: publisherCredentials,
      outDir: args.out,
    });
    if (isSuccess(apiResult.status)) {
      apiResult.oa_status = "not_checked_api_first";
      await attachSupportingInformation(apiResult, doi, args, context);
      return decorateResult(apiResult, article, {
        provider: initialRoute.provider,
        reason: initialRoute.reason,
      }, args.si);
    }
  }

  if (!args.route || args.route === "open_access") {
    const oa = await downloadOpenAccessArticle(article, { email: context.contactEmail, outDir: args.out });
    oaResult = oa;
    if (isSuccess(oa.status)) {
      if (apiResult) oa.api_attempt = attemptSummary(apiResult, publisherProvider);
      await attachSupportingInformation(oa, doi, args, context);
      return decorateResult(oa, article, {
        provider: "open_access",
        reason: apiResult ? "publisher_api_failed_oa_fallback" : "article_level_oa",
      }, args.si);
    }
    if (args.route === "open_access") return decorateResult(oa, article, { provider: "open_access", reason: "explicit_override" }, args.si);
  }

  const route = chooseRoute({
    ...article,
    isOa: false,
    routeOverride: args.route,
    hasPublisherCredentials: hasUsablePublisherCredentials(publisherProvider, publisherCredentials),
  });
  if (route.provider === "web_access") {
    const result = await runWebAccess(doi, article, args, context);
    return decorateResult({
      ...result,
      provider: "web_access",
      accessMode: "institution_browser",
      ...(oaResult ? { oa_attempt: { status: oaResult.status, assessment: oaResult.oaAssessment || "unknown" } } : {}),
    }, article, route, args.si);
  }

  const credentials = route.provider === publisherProvider ? publisherCredentials : providerCredentials(route.provider);
  if (!apiResult) {
    apiResult = await downloadPublisherArticle(article, {
      provider: route.provider,
      credentials,
      outDir: args.out,
    });
  }
  if (isSuccess(apiResult.status)) {
    await attachSupportingInformation(apiResult, doi, args, context);
    return decorateResult(apiResult, article, route, args.si);
  }
  const oaAttempt = oaResult
    ? { status: oaResult.status, assessment: oaResult.oaAssessment || "unknown" }
    : null;
  if (!apiResult.fallbackConfirmationRequired) {
    return decorateResult({ ...apiResult, ...(oaAttempt ? { oa_attempt: oaAttempt } : {}) }, article, route, args.si);
  }
  const providerFallback = args.apiFallbackWebFor?.includes(route.provider)
    ? true
    : args.noApiFallbackWebFor?.includes(route.provider)
      ? false
      : args.apiFallbackWeb;
  if (providerFallback === undefined) {
    return decorateResult({
      ...apiResult,
      api_status: apiResult.status,
      ...(oaAttempt ? { oa_attempt: oaAttempt } : {}),
      status: STATUS.API_FALLBACK_CONFIRMATION_REQUIRED,
      next_action: `Publisher API and OA fallback failed. Ask once for ${route.provider}, then re-run with --api-fallback-web-for ${route.provider} or --no-api-fallback-web-for ${route.provider}`,
    }, article, route, args.si);
  }
  if (!providerFallback) return decorateResult({ ...apiResult, ...(oaAttempt ? { oa_attempt: oaAttempt } : {}), web_fallback: "declined" }, article, route, args.si);
  const web = await runWebAccess(doi, article, args, context);
  return decorateResult({
    ...web,
    provider: "web_access",
    accessMode: "institution_browser",
    api_attempt: attemptSummary(apiResult, route.provider),
    ...(oaAttempt ? { oa_attempt: oaAttempt } : {}),
    web_fallback: "accepted",
  }, article, route, args.si);
}

function outputAndManifest(args, results, t0) {
  const secs = ((Date.now() - t0) / 1000).toFixed(1);
  const summary = { total: results.length, downloaded: results.filter((result) => isSuccess(result.status)).length, seconds: Number(secs) };
  const manifest = writeManifest(args.out, {
    request: {
      topic: args.topic || "",
      title: args.title || "",
      dois: args.dois || [],
      pdf_url: args.pdfUrl || "",
      si_requested: args.si,
      language_override: args.language || "",
      route_override: args.route || "",
      source_url: args.sourceUrl || "",
      api_fallback_web: args.apiFallbackWeb ?? null,
      api_fallback_web_for: args.apiFallbackWebFor || [],
      no_api_fallback_web_for: args.noApiFallbackWebFor || [],
    },
    summary,
    results,
  });
  const output = { summary, manifest, results };
  console.log(JSON.stringify(output, null, 2));
  return output;
}

async function main() {
  const args = parseArgs(process.argv);
  const siChoice = parseSiChoice(args);
  if (!siChoice.confirmed) {
    console.log(JSON.stringify({
      summary: { total: 0, downloaded: 0, seconds: 0 },
      results: [{ status: STATUS.SI_CONFIRMATION_REQUIRED, next_action: "Ask whether to download Supporting Information, then re-run with --si or --no-si" }],
    }, null, 2));
    process.exitCode = 2;
    return;
  }
  fs.mkdirSync(args.out, { recursive: true });
  const schoolConfig = loadSchoolConfig();
  const settings = loadSettings();
  const discoveryUrl = discoveryUrlFromConfig(schoolConfig);
  const cnkiUrl = cnkiUrlFromConfig(schoolConfig, args.cnkiUrl);
  const contactEmail = args.unpaywallEmail || settings?.open_access?.contact_email || process.env.UNPAYWALL_EMAIL || "";
  if (args.openAccess && !args.route) args.route = "open_access";
  process.stderr.write(`[config] ${schoolSummary(schoolConfig)}; discovery=${discoveryUrl}; cnki=${cnkiUrl}\n`);
  const results = [];
  const t0 = Date.now();
  let browserReady = false;
  const ensureBrowser = async () => {
    if (!browserReady) {
      await healthCheck(args.proxy);
      browserReady = true;
    }
  };
  const context = { ensureBrowser, cnkiUrl, contactEmail, singleDoi: (args.dois || []).length === 1 };

  context.cnkiTransport = createCnkiTransport({
    mode: schoolConfig?.discovery?.cnki_transport?.mode || "browser",
    entryUrl: cnkiUrl,
    webvpnUrlTemplate: schoolConfig?.discovery?.cnki_transport?.webvpn_url_template || "",
  });

  if (args.pdfUrl) {
    const directRoute = chooseDirectUrlRoute(args);
    if (directRoute.mode === "title_search") {
      const cnki = await downloadCnkiByTitle(args, context);
      results.push({ ...cnki, route: "cnki", route_reason: directRoute.reason, si_requested: args.si });
      outputAndManifest(args, results, t0);
      return;
    }
    if (directRoute.provider === "cnki") {
      await ensureBrowser();
      const cnki = await downloadCnkiDirectUrl(args.proxy, args.pdfUrl, args.title || "cnki-paper", args.out, { debug: args.debug, wantSi: args.si });
      results.push({ ...cnki, route: "cnki", route_reason: directRoute.reason, si_requested: args.si });
      outputAndManifest(args, results, t0);
      return;
    }
    const r = await downloadDirectUrl(args.pdfUrl, args.out, args.title || "").catch((e) => ({
      title: args.title || "",
      status: STATUS.FAILED_AFTER_RETRY,
      url: args.pdfUrl,
      err: String(e).slice(0, 120),
    }));
    results.push({ ...r, si_requested: args.si, ...(args.si ? { si: { status: "not_found", reason: "A direct PDF URL does not identify a supporting-information landing page" } } : {}) });
    outputAndManifest(args, results, t0);
    return;
  }

  if (args.title && (isChineseLiterature({ title: args.title, language: args.language, sourceUrl: args.sourceUrl }) || args.route === "cnki") && !args.topic) {
    const r = await downloadCnkiByTitle(args, context);
    results.push({ ...r, route: "cnki", route_reason: "chinese_literature", si_requested: args.si });
    outputAndManifest(args, results, t0);
    return;
  }

  let dois = args.dois || [];

  if (!dois.length && args.title && !args.topic) {
    if (args.openAccess) {
      const arxiv = await downloadArxivTitle(args.title, args);
      if (arxiv) {
        results.push(arxiv);
        outputAndManifest(args, results, t0);
        return;
      }
    }
    const metadata = await findCrossrefByTitle(args.title, { mailto: contactEmail }).catch(() => null);
    if (metadata?.doi) {
      dois = [metadata.doi];
      context.singleDoi = true;
    } else {
      process.stderr.write(`[oa] no exact Crossref record; trying arXiv exact title: ${args.title}\n`);
      const arxiv = await downloadArxivTitle(args.title, args);
      if (arxiv) {
        results.push(arxiv);
      } else {
        results.push({ title: args.title, status: args.openAccess ? STATUS.OA_NOT_FOUND : STATUS.METADATA_AMBIGUOUS, err: "no exact Crossref or arXiv title match", si_requested: args.si });
      }
      outputAndManifest(args, results, t0);
      return;
    }
  }

  if (!dois.length && args.topic) {
    await ensureBrowser();
    const searchQuery = wosSearchQuery(args.topic, args.title || "");
    process.stderr.write(`[wos] searching: ${searchQuery}\n`);
    const { target, urls } = await wosRecordUrls(args.proxy, searchQuery, args.count, args.debug, discoveryUrl);
    process.stderr.write(`[wos] ${urls.length} records\n`);
    for (const u of urls) {
      if (dois.length >= args.count) break;
      const doi = await doiFromRecord(args.proxy, target, u);
      if (doi && /^10\./.test(doi)) {
        dois.push(doi);
        process.stderr.write(`[doi] ${doi}\n`);
      }
    }
    if (!dois.length && args.title) {
      process.stderr.write(`[oa] WoS produced no DOI; trying arXiv exact title: ${args.title}\n`);
      const hit = await findArxivByTitle(args.title).catch((e) => ({ err: String(e).slice(0, 120) }));
      if (hit && hit.pdfUrl) {
        process.stderr.write(`[oa] arXiv ${hit.id} -> ${hit.pdfUrl}\n`);
        const r = await downloadDirectUrl(hit.pdfUrl, args.out, hit.title).catch((e) => ({
          title: args.title,
          status: STATUS.FAILED_AFTER_RETRY,
          err: String(e).slice(0, 120),
        }));
        results.push({ ...r, arxiv: hit.id });
      } else {
        results.push({ title: args.title, status: STATUS.NO_AUTHORIZED_PDF_FOUND, err: hit?.err || "no exact arXiv title match" });
      }
    }
  }
  dois = selectBatchDois(dois, args.count);

  for (const doi of dois) {
    const r = await routeDoiDownload(doi, args, context).catch((e) => {
      // Distinguish parameter/logic errors (do_not_auto_retry) from
      // network/CDP errors (failed_after_retry).
      const msg = String(e).slice(0, 120);
      const isLogic = /unknown arg|mutually exclusive|required|not reachable/i.test(msg);
      return {
        doi,
        status: isLogic ? STATUS.DO_NOT_AUTO_RETRY : STATUS.FAILED_AFTER_RETRY,
        err: msg,
      };
    });
    // Apply legacy status mapping for backward-compatible output if requested.
    if (args.legacyStatus) {
      r.status = reverseMapStatus(r.status);
    }
    results.push(r);
    process.stderr.write(
      `[dl] ${doi} -> ${r.status}${r.bytes ? " " + r.bytes + "B" : ""}\n`
    );
  }
  outputAndManifest(args, results, t0);
}

// Reverse mapping: canonical -> legacy (only for --legacy-status output).
// Best-effort; some canonical codes have no legacy equivalent and pass through.
function reverseMapStatus(s) {
  const m = {
    [STATUS.CARSI_WAITING_USER]: "needs_user_login",
    [STATUS.PUBLISHER_VERIFICATION_WAITING_USER]: "needs_user_verify",
    [STATUS.SCIENCEDIRECT_ROBOT_CHECK]: "needs_user_verify",
    [STATUS.PUBLISHER_BLOCKED_WAITING_USER]: "publisher_blocked",
    [STATUS.NO_FULL_TEXT_LINK]: "no_pdf_link",
    [STATUS.NO_AUTHORIZED_PDF_FOUND]: "no_pdf_link",
    [STATUS.FAILED_AFTER_RETRY]: "error",
    [STATUS.DO_NOT_AUTO_RETRY]: "error",
  };
  return m[s] || s;
}

if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
  main().catch((e) => {
    console.error(e.stack || String(e));
    process.exit(1);
  });
}

#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import {
  DEFAULT_PROXY,
  healthCheck,
  proxyGet,
  proxyEval,
  proxyPostUrl,
  waitForComplete,
  closeTab,
} from "./lib/cdp-utils.mjs";
import { isPdfHead } from "./lib/pdf-utils.mjs";

function usage() {
  console.log(`Usage:
  node browser_pdf_downloader.mjs --url <pdf-url> --out <file.pdf> [--proxy http://127.0.0.1:3456] [--close] [--allow-non-pdf]
  node browser_pdf_downloader.mjs --target <targetId> --out <file.pdf> [--proxy http://127.0.0.1:3456]

Downloads a PDF through an already-authenticated Chrome page controlled by the web-access CDP proxy.
It does not bypass logins, CAPTCHA, Cloudflare, paywalls, or publisher restrictions.`);
}

function parseArgs(argv) {
  const args = {
    proxy: DEFAULT_PROXY,
    close: false,
    allowNonPdf: false,
    chunkSize: 262144,
  };
  for (let i = 2; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--help" || a === "-h") {
      args.help = true;
    } else if (a === "--url") {
      args.url = argv[++i];
    } else if (a === "--target") {
      args.target = argv[++i];
    } else if (a === "--out") {
      args.out = argv[++i];
    } else if (a === "--proxy") {
      args.proxy = argv[++i].replace(/\/$/, "");
    } else if (a === "--close") {
      args.close = true;
    } else if (a === "--allow-non-pdf") {
      args.allowNonPdf = true;
    } else if (a === "--chunk-size") {
      args.chunkSize = Number(argv[++i]);
    } else {
      throw new Error(`Unknown argument: ${a}`);
    }
  }
  return args;
}

async function main() {
  const args = parseArgs(process.argv);
  if (args.help) {
    usage();
    return;
  }
  if (!args.out) throw new Error("--out is required");
  if (!args.url && !args.target) throw new Error("Provide --url or --target");

  // Fail fast with a friendly message if the CDP proxy isn't running.
  await healthCheck(args.proxy);

  let target = args.target;
  let openedByScript = false;

  if (!target) {
    const created = await proxyPostUrl(args.proxy, "/new", args.url, {}, 60000);
    target = created.targetId;
    openedByScript = true;
    await waitForComplete(args.proxy, target);
  } else if (args.url) {
    await proxyPostUrl(args.proxy, "/navigate", args.url, { target }, 60000);
    await waitForComplete(args.proxy, target);
  }

  // FIX: fetch the *requested* URL, not location.href — the page may have
  // JS-redirected after navigation, and fetching location.href would grab the
  // redirected HTML page instead of the PDF. Fall back to location.href only
  // when --target was given without --url (caller already navigated).
  const fetchTarget = args.url ? JSON.stringify(args.url) : "location.href";
  const initJs = `(
    async () => {
      const r = await fetch(${fetchTarget}, { credentials: "include" });
      const ct = r.headers.get("content-type") || "";
      const ab = await r.arrayBuffer();
      window.__natureDownloaderBytes = new Uint8Array(ab);
      return {
        ok: r.ok,
        status: r.status,
        contentType: ct,
        size: window.__natureDownloaderBytes.length,
        url: location.href,
        head: Array.from(window.__natureDownloaderBytes.slice(0, 8))
      };
    }
  )()`;

  const init = await proxyEval(args.proxy, target, initJs, 120000);
  const meta = init.value;
  if (!meta || !meta.ok) {
    throw new Error(`Browser fetch failed: ${JSON.stringify(meta)}`);
  }

  const headBytes = Buffer.from(meta.head || []);
  if (!args.allowNonPdf && !isPdfHead(headBytes)) {
    // Clean up the window var before erroring out.
    await proxyEval(args.proxy, target, `delete window.__natureDownloaderBytes`, 10000).catch(() => {});
    throw new Error(
      `Downloaded content is not a PDF. content-type=${meta.contentType}, head=${JSON.stringify(meta.head)}. ` +
        `If this is expected, rerun with --allow-non-pdf.`
    );
  }

  fs.mkdirSync(path.dirname(path.resolve(args.out)), { recursive: true });
  const stream = fs.createWriteStream(args.out);
  const size = Number(meta.size);
  try {
    for (let start = 0; start < size; start += args.chunkSize) {
      const end = Math.min(start + args.chunkSize, size);
      const chunkJs = `(
        () => {
          const bytes = window.__natureDownloaderBytes.slice(${start}, ${end});
          let bin = "";
          for (let i = 0; i < bytes.length; i += 0x8000) {
            bin += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));
          }
          return btoa(bin);
        }
      )()`;
      const chunk = await proxyEval(args.proxy, target, chunkJs, 120000);
      stream.write(Buffer.from(chunk.value, "base64"));
    }
    await new Promise((resolve, reject) => {
      stream.end(resolve);
      stream.on("error", reject);
    });
  } finally {
    // Always clean up the window var.
    await proxyEval(args.proxy, target, `delete window.__natureDownloaderBytes`, 10000).catch(() => {});
  }

  const saved = fs.readFileSync(args.out);
  const savedHead = saved.subarray(0, 8).toString("ascii");
  const result = {
    out: path.resolve(args.out),
    bytes: saved.length,
    contentType: meta.contentType,
    sourceUrl: meta.url,
    signature: savedHead,
    pdf: savedHead.startsWith("%PDF"),
  };
  console.log(JSON.stringify(result, null, 2));

  if (args.close && openedByScript) {
    await closeTab(args.proxy, target);
  }
}

main().catch((error) => {
  console.error(error.stack || String(error));
  process.exit(1);
});

#!/usr/bin/env node
import { DEFAULT_PROXY, healthCheck, proxyGet, proxyPostUrl, waitForComplete } from "./lib/cdp-utils.mjs";

function usage() {
  console.log(`Usage:
  node cdp_open_url.mjs --url <url> [--proxy http://127.0.0.1:3456] [--wait]

Opens a URL in the already-authorized Chrome CDP proxy.
This helper URL-encodes nested URLs correctly, which matters for URLs containing # or #! fragments.
It does not read cookies, passwords, local storage, or browser profiles.`);
}

function parseArgs(argv) {
  const args = {
    proxy: DEFAULT_PROXY,
    wait: false,
  };
  for (let i = 2; i < argv.length; i++) {
    const a = argv[i];
    if (a === "--help" || a === "-h") args.help = true;
    else if (a === "--url") args.url = argv[++i];
    else if (a === "--proxy") args.proxy = argv[++i].replace(/\/$/, "");
    else if (a === "--wait") args.wait = true;
    else throw new Error(`Unknown argument: ${a}`);
  }
  return args;
}

async function main() {
  const args = parseArgs(process.argv);
  if (args.help) {
    usage();
    return;
  }
  if (!args.url) throw new Error("--url is required");

  // Fail fast with a friendly message if the CDP proxy isn't running.
  await healthCheck(args.proxy);

  const created = await proxyPostUrl(args.proxy, "/new", args.url, {}, 60000);
  const target = created.targetId;
  const info = args.wait
    ? await waitForComplete(args.proxy, target)
    : await proxyGet(args.proxy, "/info", { target }, 10000);
  console.log(
    JSON.stringify(
      {
        targetId: target,
        title: info?.title || null,
        url: info?.url || null,
        ready: info?.ready || null,
      },
      null,
      2
    )
  );
}

main().catch((error) => {
  console.error(error.stack || String(error));
  process.exit(1);
});

#!/usr/bin/env python3
"""Configure publisher API credentials without echoing secrets."""

from __future__ import annotations

import argparse
import getpass
import json
import os
import sys
import tempfile
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path

REGISTRY_FILE = Path(__file__).resolve().parents[1] / "data" / "publishers.json"
PROVIDER_REGISTRY = json.loads(REGISTRY_FILE.read_text(encoding="utf-8"))
PROVIDERS = tuple(PROVIDER_REGISTRY)
CONFIG_DIR = Path(os.environ.get("LIT_DL_CONFIG_DIR", Path.home() / ".config" / "lit-dl"))
CREDENTIALS_FILE = CONFIG_DIR / "credentials.json"
SETTINGS_FILE = CONFIG_DIR / "settings.json"


def _load(path: Path) -> dict:
    if not path.exists():
        return {}
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except (json.JSONDecodeError, OSError):
        return {}


def _save(path: Path, value: dict, mode: int = 0o600) -> None:
    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
    fd, temp_name = tempfile.mkstemp(prefix=path.name, dir=path.parent)
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as handle:
            json.dump(value, handle, ensure_ascii=False, indent=2)
            handle.write("\n")
        os.chmod(temp_name, mode)
        os.replace(temp_name, path)
        os.chmod(path, mode)
    finally:
        if os.path.exists(temp_name):
            os.unlink(temp_name)


def _mask(value: str) -> str:
    return "*" * max(0, len(value) - 4) + value[-4:]


def cmd_set(args: argparse.Namespace) -> int:
    if args.stdin:
        api_key = sys.stdin.readline().strip()
    else:
        api_key = args.api_key or os.environ.get("LIT_DL_API_KEY") or getpass.getpass("API key: ")
    if not api_key:
        raise SystemExit("API key 不能为空")
    values = {"api_key": api_key.strip()}
    if args.insttoken:
        values["insttoken"] = args.insttoken.strip()
    if args.authtoken:
        values["authtoken"] = args.authtoken.strip()
    if args.fulltext_endpoint:
        values["fulltext_endpoint"] = args.fulltext_endpoint.strip()
    all_credentials = _load(CREDENTIALS_FILE)
    all_credentials[args.provider] = values
    _save(CREDENTIALS_FILE, all_credentials)
    print(json.dumps({"ok": True, "provider": args.provider, "path": str(CREDENTIALS_FILE), "api_key": _mask(values["api_key"])}, ensure_ascii=False))
    return 0


def cmd_show(_: argparse.Namespace) -> int:
    credentials = _load(CREDENTIALS_FILE)
    masked = {provider: {key: _mask(str(value)) for key, value in values.items()} for provider, values in credentials.items()}
    print(json.dumps({"path": str(CREDENTIALS_FILE), "credentials": masked, "settings": _load(SETTINGS_FILE)}, ensure_ascii=False, indent=2))
    return 0


def cmd_delete(args: argparse.Namespace) -> int:
    credentials = _load(CREDENTIALS_FILE)
    credentials.pop(args.provider, None)
    _save(CREDENTIALS_FILE, credentials)
    print(json.dumps({"ok": True, "provider": args.provider}, ensure_ascii=False))
    return 0


def _validation_request(provider: str, values: dict) -> urllib.request.Request:
    key = values["api_key"]
    if provider == "elsevier":
        url = "https://api.elsevier.com/content/article/doi/10.1016/S0140-6736(20)30183-5?view=META"
        headers = {"X-ELS-APIKey": key, "Accept": "application/json"}
        if values.get("insttoken"):
            headers["X-ELS-Insttoken"] = values["insttoken"]
        return urllib.request.Request(url, headers=headers)
    if provider == "springer_nature":
        url = "https://api.springernature.com/meta/v2/json?" + urllib.parse.urlencode({"q": "keyword:test", "api_key": key, "p": 1})
        return urllib.request.Request(url, headers={"Accept": "application/json"})
    url = "https://ieeexploreapi.ieee.org/api/v1/search/articles?" + urllib.parse.urlencode({"apikey": key, "querytext": "test", "max_records": 1})
    return urllib.request.Request(url, headers={"Accept": "application/json"})


def cmd_validate(args: argparse.Namespace) -> int:
    values = _load(CREDENTIALS_FILE).get(args.provider)
    if not values or not values.get("api_key"):
        print(json.dumps({"ok": False, "provider": args.provider, "status": "credentials_missing"}, ensure_ascii=False))
        return 2
    try:
        with urllib.request.urlopen(_validation_request(args.provider, values), timeout=30) as response:
            status = response.status
    except urllib.error.HTTPError as error:
        status = error.code
    except OSError as error:
        print(json.dumps({"ok": False, "provider": args.provider, "status": "validation_failed", "error": str(error)}, ensure_ascii=False))
        return 1
    valid = status not in (401, 403)
    print(json.dumps({"ok": valid, "provider": args.provider, "http_status": status, "status": "configured" if valid else "credentials_invalid"}, ensure_ascii=False))
    return 0 if valid else 1


def cmd_contact_email(args: argparse.Namespace) -> int:
    settings = _load(SETTINGS_FILE)
    settings.setdefault("open_access", {})["contact_email"] = args.email
    _save(SETTINGS_FILE, settings)
    print(json.dumps({"ok": True, "path": str(SETTINGS_FILE), "contact_email": args.email}, ensure_ascii=False))
    return 0


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Configure nature-downloader publisher API credentials.")
    sub = parser.add_subparsers(dest="command", required=True)
    set_cmd = sub.add_parser("set")
    set_cmd.add_argument("provider", choices=PROVIDERS)
    set_cmd.add_argument("--api-key", help="Prefer the hidden prompt or LIT_DL_API_KEY over command-line secrets.")
    set_cmd.add_argument("--stdin", action="store_true", help="Read the API key from stdin without placing it in process arguments or output.")
    set_cmd.add_argument("--insttoken", help="Elsevier institutional token, if issued.")
    set_cmd.add_argument("--authtoken", help="Elsevier authentication token, if issued.")
    set_cmd.add_argument("--fulltext-endpoint", help="IEEE paid Full-Text Access API endpoint template issued for your product; use {doi} as the placeholder.")
    set_cmd.set_defaults(func=cmd_set)
    show = sub.add_parser("show")
    show.set_defaults(func=cmd_show)
    delete = sub.add_parser("delete")
    delete.add_argument("provider", choices=PROVIDERS)
    delete.set_defaults(func=cmd_delete)
    validate = sub.add_parser("validate")
    validate.add_argument("provider", choices=PROVIDERS)
    validate.set_defaults(func=cmd_validate)
    email = sub.add_parser("contact-email")
    email.add_argument("email")
    email.set_defaults(func=cmd_contact_email)
    return parser


def main() -> int:
    args = build_parser().parse_args()
    return args.func(args)


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Command line helper for literature-downloader school configuration."""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
if str(SRC) not in sys.path:
    sys.path.insert(0, str(SRC))

from config import CONFIG_FILE, config_exists, load_config, save_config, validate  # noqa: E402
from health_check import health_check  # noqa: E402
from schools_loader import list_school_names  # noqa: E402
from wizard import Wizard, infer_access_from_url  # noqa: E402


def cmd_preset(args: argparse.Namespace) -> int:
    result = Wizard().configure_from_preset(args.school)
    cfg = result["config"]
    print(json.dumps({"ok": True, "path": result["path"], "school": cfg["school"]["name"]}, ensure_ascii=False))
    return 0


def cmd_url(args: argparse.Namespace) -> int:
    result = Wizard().configure_from_resource_url(args.url)
    cfg = result["config"]
    inferred = result["inferred"]
    print(
        json.dumps(
            {
                "ok": True,
                "path": result["path"],
                "school": cfg["school"]["name"],
                "entry_type": inferred["entry_type"],
                "auth_type": inferred["auth_type"],
                "sso_domain": inferred["sso_domain"],
                "resource_entry": inferred["resource_entry"],
            },
            ensure_ascii=False,
        )
    )
    return 0


def cmd_infer(args: argparse.Namespace) -> int:
    print(json.dumps(infer_access_from_url(args.url), ensure_ascii=False, indent=2))
    return 0


def cmd_show(_: argparse.Namespace) -> int:
    cfg = load_config()
    if cfg is None:
        print(f"尚未配置，配置文件路径：{CONFIG_FILE}")
        return 2
    errors = validate(cfg)
    print(json.dumps({"ok": not errors, "path": str(CONFIG_FILE), "errors": errors, "config": cfg}, ensure_ascii=False, indent=2))
    return 0 if not errors else 1


def cmd_cnki_url(args: argparse.Namespace) -> int:
    cfg = load_config()
    if cfg is None:
        print(f"尚未配置，配置文件路径：{CONFIG_FILE}")
        return 2
    cfg.setdefault("discovery", {})["cnki_url"] = args.url
    path = save_config(cfg)
    print(json.dumps({"ok": True, "path": str(path), "cnki_url": args.url}, ensure_ascii=False))
    return 0


def cmd_health(args: argparse.Namespace) -> int:
    result = health_check(force=args.force)
    print(json.dumps(result, ensure_ascii=False, indent=2))
    return 0 if result.get("ok") else 1


def cmd_list(_: argparse.Namespace) -> int:
    for name in list_school_names():
        print(name)
    return 0


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Configure school access for nature-downloader.")
    sub = parser.add_subparsers(dest="command", required=True)

    preset = sub.add_parser("preset", help="Configure from bundled school preset.")
    preset.add_argument("school", help="Optional user-maintained preset name. The distributed skill contains no institution presets.")
    preset.set_defaults(func=cmd_preset)

    url = sub.add_parser("url", help="Configure from a library resource portal or authentication URL.")
    url.add_argument("url", help="Library resource/database portal URL.")
    url.set_defaults(func=cmd_url)

    infer = sub.add_parser("infer", help="Infer access route from a library resource URL without saving.")
    infer.add_argument("url", help="Library resource/database portal URL.")
    infer.set_defaults(func=cmd_infer)

    show = sub.add_parser("show", help="Show current configuration.")
    show.set_defaults(func=cmd_show)

    cnki_url = sub.add_parser("cnki-url", help="Save the CNKI entry URL for Chinese literature downloads.")
    cnki_url.add_argument("url", help="CNKI entry URL from the library portal, or the public CNKI search entry.")
    cnki_url.set_defaults(func=cmd_cnki_url)

    health = sub.add_parser("health", help="Run connectivity health check.")
    health.add_argument("--force", action="store_true", help="Ignore cached health result.")
    health.set_defaults(func=cmd_health)

    list_cmd = sub.add_parser("list", help="List bundled school presets.")
    list_cmd.set_defaults(func=cmd_list)

    return parser


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)
    return args.func(args)


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
import argparse
import json
import sys
from pathlib import Path


def configure_utf8_stdio():
    """Keep Chinese paths/text printable in Windows PowerShell and Claude Code terminals."""
    for stream_name in ("stdout", "stderr"):
        stream = getattr(sys, stream_name, None)
        if hasattr(stream, "reconfigure"):
            try:
                stream.reconfigure(encoding="utf-8", errors="replace")
            except Exception:
                pass


def extract_with_pdfplumber(pdf_path, max_pages):
    import pdfplumber

    chunks = []
    with pdfplumber.open(str(pdf_path)) as pdf:
        page_count = len(pdf.pages)
        pages = pdf.pages if max_pages is None else pdf.pages[:max_pages]
        for i, page in enumerate(pages, start=1):
            chunks.append(f"\n\n===== PAGE {i} =====\n")
            chunks.append(page.extract_text() or "")
    return page_count, "".join(chunks)


def extract_with_pypdf(pdf_path, max_pages):
    from pypdf import PdfReader

    reader = PdfReader(str(pdf_path))
    page_count = len(reader.pages)
    pages = reader.pages if max_pages is None else reader.pages[:max_pages]
    chunks = []
    for i, page in enumerate(pages, start=1):
        chunks.append(f"\n\n===== PAGE {i} =====\n")
        chunks.append(page.extract_text() or "")
    return page_count, "".join(chunks)


def main():
    configure_utf8_stdio()

    parser = argparse.ArgumentParser(description="Extract text from a downloaded PDF for verification or reading.")
    parser.add_argument("--pdf", required=True, help="Path to PDF")
    parser.add_argument("--out", help="Optional output .txt path")
    parser.add_argument("--pages", type=int, default=3, help="Number of pages to extract; use 0 for all pages")
    parser.add_argument("--json", action="store_true", help="Print JSON metadata instead of text preview")
    args = parser.parse_args()

    pdf_path = Path(args.pdf)
    max_pages = None if args.pages == 0 else args.pages

    try:
        page_count, text = extract_with_pdfplumber(pdf_path, max_pages)
        engine = "pdfplumber"
    except Exception:
        page_count, text = extract_with_pypdf(pdf_path, max_pages)
        engine = "pypdf"

    if args.out:
        out = Path(args.out)
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_text(text, encoding="utf-8")

    meta = {
        "pdf": str(pdf_path.resolve()),
        "pages_total": page_count,
        "pages_extracted": page_count if max_pages is None else min(max_pages, page_count),
        "chars": len(text),
        "engine": engine,
        "out": str(Path(args.out).resolve()) if args.out else None,
    }

    if args.json:
        print(json.dumps(meta, ensure_ascii=False, indent=2))
    else:
        print(json.dumps(meta, ensure_ascii=False))
        print(text[:4000])


if __name__ == "__main__":
    main()

// Automatic verification challenge handler for the literature downloader.
//
// Attempts to automatically pass common publisher verification challenges
// (slider CAPTCHAs, robot checks, Cloudflare Turnstile, simple click-to-continue)
// before falling back to user intervention. Operates entirely within the user's
// existing authenticated Chrome session via CDP — never reads credentials.
//
// Strategy priority (ordered by success rate):
//   1. Simple click challenges  ("Continue" / "Verify" / 确认  buttons)
//   2. ScienceDirect robot check (checkbox-style verification)
//   3. Cloudflare Turnstile      (managed-mode checkbox)
//   4. Slider / drag CAPTCHAs    (CNKI Geetest-style, generic drag-to-end)
//   5. reCAPTCHA / image CAPTCHA (NOT auto-solvable — returns false)
//
// Each strategy returns { passed: true } on success, { passed: false } on failure.
// The caller should try strategies sequentially and only hand off to the user
// when all strategies fail.

import { evalJs, sleep, proxyGet, waitForComplete } from "./cdp-utils.mjs";
import { STATUS } from "./status-codes.mjs";

// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------

/**
 * Attempt to automatically pass any detected verification challenge.
 *
 * @param {string} proxy   - CDP proxy URL (e.g. http://127.0.0.1:3456)
 * @param {string} target  - Chrome target/tab ID
 * @param {object} wallInfo - Result from classifyWall() { status, reason }
 * @param {object} opts    - { debug?: boolean, maxAttempts?: number }
 * @returns {Promise<{passed: boolean, attempted: boolean, method?: string, status?: string}>}
 */
export async function handleVerification(proxy, target, wallInfo = {}, opts = {}) {
  const debug = opts.debug || false;
  const maxAttempts = opts.maxAttempts || 2;

  // Only attempt auto-verification for known challenge types.
  const autoAttemptable = [
    STATUS.PUBLISHER_VERIFICATION_WAITING_USER,
    STATUS.SCIENCEDIRECT_ROBOT_CHECK,
    STATUS.PUBLISHER_BLOCKED_WAITING_USER,
  ];
  if (wallInfo.status && !autoAttemptable.includes(wallInfo.status)) {
    if (debug) process.stderr.write(`[anti-bot] status ${wallInfo.status} not auto-attemptable, skipping\n`);
    return { passed: false, attempted: false, status: wallInfo.status };
  }

  // Get page context for targeted strategy selection
  const snap = await pageSnapshot(proxy, target);
  if (debug) {
    process.stderr.write(
      `[anti-bot] wall=${wallInfo.status} url=${(snap.url||'').slice(0,100)} ` +
      `title=${(snap.title||'').slice(0,80)}\n`
    );
  }

  const strategies = rankStrategies(snap, wallInfo);

  for (let attempt = 0; attempt < maxAttempts; attempt++) {
    for (const strategy of strategies) {
      if (debug) process.stderr.write(`[anti-bot] trying ${strategy.name} (attempt ${attempt+1})\n`);
      try {
        const result = await strategy.fn(proxy, target, snap, debug);
        if (result.passed) {
          if (debug) process.stderr.write(`[anti-bot] ✓ ${strategy.name} passed\n`);
          return { ...result, attempted: true };
        }
      } catch (e) {
        if (debug) process.stderr.write(`[anti-bot] ✗ ${strategy.name} error: ${String(e).slice(0,80)}\n`);
      }
    }
    if (attempt < maxAttempts - 1) await sleep(2000);
  }

  if (debug) process.stderr.write(`[anti-bot] all strategies failed after ${maxAttempts} attempts\n`);
  return { passed: false, attempted: true };
}

// ---------------------------------------------------------------------------
// Strategy ranking
// ---------------------------------------------------------------------------

function rankStrategies(snap, wallInfo) {
  const url = (snap.url || "").toLowerCase();
  const text = `${snap.title||""} ${snap.body||""}`.toLowerCase();

  const all = [
    { name: "simple_click", fn: trySimpleClickChallenge, priority: 1 },
    { name: "sciencedirect_robot", fn: tryScienceDirectRobotCheck, priority: 2 },
    { name: "cloudflare_turnstile", fn: tryCloudflareTurnstile, priority: 3 },
    { name: "slider_captcha", fn: trySliderCaptcha, priority: 4 },
    { name: "cnki_slider", fn: tryCNKISlider, priority: 4 },
    { name: "recaptcha_bypass", fn: tryRecaptchaBypass, priority: 5 },
  ];

  // Boost ScienceDirect strategy when we know we're on their domain
  if (/sciencedirect\.com|sciencedirect\.elsevier/i.test(url)) {
    all.find(s => s.name === "sciencedirect_robot").priority = 0;
  }
  // Boost CNKI slider when on CNKI domain
  if (/cnki\.net|cnki\.com\.cn/i.test(url)) {
    all.find(s => s.name === "cnki_slider").priority = 0;
  }
  // Boost Cloudflare when we see Cloudflare in text
  if (/cloudflare|ray id|checking your browser/i.test(text)) {
    all.find(s => s.name === "cloudflare_turnstile").priority = 0;
  }
  // Boost slider when we see slider/滑块 keywords
  if (/滑块|滑动验证|drag|slide|拼图|puzzle/i.test(text)) {
    all.find(s => s.name === "cnki_slider").priority = 0;
    all.find(s => s.name === "slider_captcha").priority = 1;
  }

  return all.sort((a, b) => a.priority - b.priority);
}

// ---------------------------------------------------------------------------
// Strategy 1: Simple Click Challenge
// Clicks buttons like "Continue", "Verify", "Proceed", "确认", "验证"
// ---------------------------------------------------------------------------

async function trySimpleClickChallenge(proxy, target, snap, debug) {
  const clicked = await evalJs(proxy, target, `(()=>{
    const buttons = [
      ...document.querySelectorAll('button, input[type="button"], input[type="submit"], a.btn, a.button, [role="button"]')
    ];
    const textPatterns = [
      /^\\s*(继续|确认|验证|确定|提交|Continue|Verify|Proceed|Submit|Next|OK|Go|I am not a robot|I'm not a robot)\\s*$/i,
      /继续访问|确认提交|开始验证|立即验证|确认并继续/i,
    ];
    for (const b of buttons) {
      const txt = (b.innerText || b.value || b.title || b.getAttribute('aria-label') || '').trim();
      if (txt.length > 30) continue; // skip long text
      for (const re of textPatterns) {
        if (re.test(txt)) {
          b.click();
          return JSON.stringify({clicked:true, text:txt, tag:b.tagName});
        }
      }
    }
    // Also try links with verification-related text
    const links = [...document.querySelectorAll('a')];
    for (const a of links) {
      const txt = (a.innerText || a.title || '').trim();
      if (txt.length > 30) continue;
      for (const re of textPatterns) {
        if (re.test(txt)) {
          a.click();
          return JSON.stringify({clicked:true, text:txt, tag:'A'});
        }
      }
    }
    return JSON.stringify({clicked:false});
  })`, 15000);

  if (!clicked || !JSON.parse(clicked).clicked) return { passed: false };

  // Wait and check if the challenge was resolved
  await sleep(3000);
  await waitForComplete(proxy, target, 15000);
  const after = await pageSnapshot(proxy, target);
  const stillBlocked = await evalJs(proxy, target, `(()=>{
    const txt = (document.title||'') + ' ' + (document.body?.innerText||'').slice(0,500);
    return /captcha|robot|cloudflare|verify|challenge|验证|滑块|人机验证/i.test(txt) &&
      !/search results|article|download|pdf/i.test(txt);
  })`, 10000).catch(() => true);

  if (!stillBlocked) {
    return {
      passed: true,
      method: "simple_click",
      newUrl: after.url,
    };
  }
  return { passed: false };
}

// ---------------------------------------------------------------------------
// Strategy 2: ScienceDirect Robot Check
// Handles SD's "Are you a robot?" page — typically a checkbox click
// ---------------------------------------------------------------------------

async function tryScienceDirectRobotCheck(proxy, target, snap, debug) {
  // ScienceDirect robot check: find the challenge iframe or checkbox and interact
  const result = await evalJs(proxy, target, `(()=>{
    // Try to find and click the "I am not a robot" / checkbox element
    // SD often embeds a challenge iframe or has a specific verification element

    // 1. Look for SD-specific verify button
    const verifyBtn = document.querySelector('#verify, .verify-button, [data-verify], button[type="submit"]');
    if (verifyBtn && /verify|robot|challenge/i.test(verifyBtn.innerText || verifyBtn.value || '')) {
      verifyBtn.click();
      return JSON.stringify({method:'sd_verify_btn', clicked:true});
    }

    // 2. Look for reCAPTCHA iframe and try to click the checkbox inside
    const frames = document.querySelectorAll('iframe[src*="recaptcha"], iframe[src*="captcha"], iframe[title*="recaptcha"], iframe[title*="widget"]');
    for (const frame of frames) {
      try {
        const innerDoc = frame.contentDocument || frame.contentWindow.document;
        const checkbox = innerDoc.querySelector('.recaptcha-checkbox-border, .recaptcha-checkbox, [role="checkbox"], #recaptcha-anchor');
        if (checkbox) {
          checkbox.click();
          return JSON.stringify({method:'recaptcha_checkbox', clicked:true});
        }
      } catch(e) { /* cross-origin iframe — can't access */ }
    }

    // 3. Look for Cloudflare Turnstile iframe
    const cfFrames = document.querySelectorAll('iframe[src*="cloudflare"], iframe[src*="turnstile"], iframe[src*="challenges.cloudflare"]');
    for (const frame of cfFrames) {
      try {
        const innerDoc = frame.contentDocument || frame.contentWindow.document;
        const checkbox = innerDoc.querySelector('input[type="checkbox"], [role="checkbox"], label');
        if (checkbox) {
          checkbox.click();
          return JSON.stringify({method:'cf_checkbox', clicked:true});
        }
      } catch(e) { /* cross-origin */ }
    }

    return JSON.stringify({clicked:false});
  })`, 15000);

  const parsed = JSON.parse(result || '{"clicked":false}');
  if (!parsed.clicked) return { passed: false };

  // Wait longer for SD — their verification can take 5-10 seconds
  await sleep(5000);
  await waitForComplete(proxy, target, 20000);

  // Check if we're past the robot check
  const after = await pageSnapshot(proxy, target);
  const stillBlocked = await evalJs(proxy, target, `(()=>{
    const txt = (document.title||'') + ' ' + (document.body?.innerText||'').slice(0,500);
    return /are you a robot|cloudflare|verify you are human|bot verification/i.test(txt);
  })`, 10000).catch(() => false);

  if (!stillBlocked) {
    return {
      passed: true,
      method: parsed.method || "sciencedirect_robot",
      newUrl: after.url,
    };
  }
  return { passed: false };
}

// ---------------------------------------------------------------------------
// Strategy 3: Cloudflare Turnstile
// Attempts to trigger Turnstile challenge resolution
// ---------------------------------------------------------------------------

async function tryCloudflareTurnstile(proxy, target, snap, debug) {
  // Cloudflare Turnstile can sometimes be auto-resolved in managed mode
  // by clicking the checkbox or calling turnstile.render()
  const result = await evalJs(proxy, target, `(()=>{
    // Try to find Turnstile widget and interact with it
    const widgets = document.querySelectorAll('.cf-turnstile, [data-sitekey], iframe[src*="turnstile"], iframe[src*="challenges.cloudflare"]');
    if (widgets.length === 0) return JSON.stringify({found:false});

    // Try to find and click the Turnstile checkbox inside shadow DOM or iframe
    for (const w of widgets) {
      // Check shadow DOM
      if (w.shadowRoot) {
        const cb = w.shadowRoot.querySelector('input[type="checkbox"], [role="checkbox"]');
        if (cb) { cb.click(); return JSON.stringify({found:true, clicked:true, method:'shadow_cb'}); }
      }
    }

    // Try to find iframe and click checkbox inside
    const frames = document.querySelectorAll('iframe');
    for (const frame of frames) {
      if (!/turnstile|cloudflare/i.test(frame.src || '')) continue;
      try {
        const innerDoc = frame.contentDocument || frame.contentWindow.document;
        const cb = innerDoc.querySelector('input[type="checkbox"], [role="checkbox"], label.checkbox');
        if (cb) {
          // First focus the iframe, then click
          frame.focus();
          const rect = cb.getBoundingClientRect();
          const clickEvent = new MouseEvent('click', {
            bubbles: true, cancelable: true,
            clientX: rect.left + rect.width/2,
            clientY: rect.top + rect.height/2,
          });
          cb.dispatchEvent(clickEvent);
          return JSON.stringify({found:true, clicked:true, method:'cf_iframe_cb'});
        }
      } catch(e) { /* cross-origin */ }
    }

    // Try calling turnstile callback if available
    try {
      if (window.turnstile && typeof window.turnstile.render === 'function') {
        return JSON.stringify({found:true, method:'turnstile_api', note:'render available but requires callback'});
      }
      // Check for __cf_chl_opt or similar variables
      if (window.__cf_chl_opt || window._cf_chl_opt) {
        return JSON.stringify({found:true, method:'cf_challenge', note:'challenge detected, may need manual solving'});
      }
    } catch(e) {}

    return JSON.stringify({found:true, clicked:false});
  })`, 15000);

  const parsed = JSON.parse(result || '{"found":false}');
  if (!parsed.found || !parsed.clicked) return { passed: false };

  // Wait for Turnstile to resolve
  await sleep(4000);
  await waitForComplete(proxy, target, 20000);

  const stillBlocked = await evalJs(proxy, target, `(()=>{
    const txt = (document.title||'') + ' ' + (document.body?.innerText||'').slice(0,500);
    return /checking your browser|cloudflare|ddos protection|just a moment/i.test(txt) &&
      !/article|download|pdf|search/i.test(txt);
  })`, 10000).catch(() => false);

  if (!stillBlocked) {
    return { passed: true, method: parsed.method || "cloudflare_turnstile" };
  }
  return { passed: false };
}

// ---------------------------------------------------------------------------
// Strategy 4: Generic Slider CAPTCHA
// Detects slider track + knob, calculates gap, simulates human-like drag
// ---------------------------------------------------------------------------

async function trySliderCaptcha(proxy, target, snap, debug) {
  // First, detect if there's a slider on the page
  const detection = await evalJs(proxy, target, `(()=>{
    // Common slider CAPTCHA element selectors
    const selectors = [
      '.slider-captcha', '[class*="slider"]', '[class*="captcha"]',
      '.geetest_slider', '.geetest_canvas', '.gt_slider', '.gt_slider_knob',
      '.yidun_slider', '.yidun_slider_indicator',
      '.nc_wrapper', '.nc_scale', '.scale_text', // Aliyun
      '.dx_captcha_slider', '.dx_captcha_slider_knob', // DingXiang
      '.verifybox', '.slide-verify', '.slider-verify',
      '[class*="slide-verify"]', '[class*="slideVerify"]',
      '.drag_captcha', '.drag-slider',
    ];

    const found = [];
    for (const sel of selectors) {
      const el = document.querySelector(sel);
      if (el) {
        const rect = el.getBoundingClientRect();
        found.push({
          selector: sel,
          tag: el.tagName,
          visible: rect.width > 0 && rect.height > 0,
          rect: { x:Math.round(rect.x), y:Math.round(rect.y), w:Math.round(rect.width), h:Math.round(rect.height) },
        });
      }
    }

    // Also detect slider-specific DOM structure
    const canvases = [...document.querySelectorAll('canvas')];
    const canvasInfo = canvases.map(c => ({
      w: c.width,
      h: c.height,
      rect: (()=>{const r=c.getBoundingClientRect();return {x:Math.round(r.x),y:Math.round(r.y),w:Math.round(r.width),h:Math.round(r.height)};})(),
    }));

    return JSON.stringify({found, canvasInfo, canvasCount: canvases.length});
  })`, 15000);

  const info = JSON.parse(detection || '{"found":[],"canvasInfo":[]}');
  if (debug) process.stderr.write(`[anti-bot][slider] detected: ${info.found.length} sliders, ${info.canvasCount} canvases\n`);

  // Find the most likely slider knob and track
  const knobResult = await evalJs(proxy, target, `(()=>{
    // Look for draggable slider knobs
    const knobSelectors = [
      '.gt_slider_knob', '.geetest_slider_button', '.geetest_slide_button',
      '.nc_iconfont', '.btn_slide', '.slider-button', '.slider-btn',
      '.dx_captcha_slider_knob', '.slider-knob', '[class*="slider-knob"]',
      '[class*="slider_knob"]', '[class*="slide_btn"]', '.drag-button',
      '.yidun_slider__indicator',
      '[class*="slider"] [class*="btn"]',
      '[class*="slider"] [class*="button"]',
    ];

    for (const sel of knobSelectors) {
      const el = document.querySelector(sel);
      if (el) {
        const rect = el.getBoundingClientRect();
        if (rect.width > 20 && rect.height > 20) {
          return JSON.stringify({
            knob: { selector:sel, x:Math.round(rect.x), y:Math.round(rect.y), w:Math.round(rect.width), h:Math.round(rect.height) },
          });
        }
      }
    }

    // Fallback: look for any element that contains slider-like event handlers
    const all = [...document.querySelectorAll('*')];
    for (const el of all) {
      const cls = (el.className || '').toString().toLowerCase();
      const rect = el.getBoundingClientRect();
      if (rect.width > 30 && rect.height > 30 && rect.width < 400) {
        if (/knob|slider.*btn|slide.*btn|drag.*btn/.test(cls)) {
          return JSON.stringify({
            knob: { selector:'.'+cls.replace(/\\s/g,'.'), x:Math.round(rect.x), y:Math.round(rect.y), w:Math.round(rect.width), h:Math.round(rect.height) },
          });
        }
      }
    }

    return JSON.stringify({knob:null});
  })`, 10000);

  const { knob } = JSON.parse(knobResult || '{"knob":null}');
  if (!knob) {
    if (debug) process.stderr.write('[anti-bot][slider] no knob found\n');
    return { passed: false };
  }

  if (debug) process.stderr.write(`[anti-bot][slider] knob at (${knob.x},${knob.y}) ${knob.w}x${knob.h}\n`);

  // Calculate the target distance to drag
  // Try image gap detection first (for Geetest-style), fall back to track width
  const distanceResult = await evalJs(proxy, target, `(()=>{
    // Method 1: Calculate using canvas images (Geetest-style)
    const canvases = [...document.querySelectorAll('canvas')];
    if (canvases.length >= 2) {
      // Find the background canvas (larger one, usually)
      const sorted = canvases.sort((a,b) => (b.width*b.height) - (a.width*a.height));
      const bgCanvas = sorted[0];
      const sliderCanvas = canvases.find(c => c !== bgCanvas && (c.width < bgCanvas.width));

      if (bgCanvas && sliderCanvas) {
        try {
          const bgCtx = bgCanvas.getContext('2d');
          const bgData = bgCtx.getImageData(0, 0, bgCanvas.width, bgCanvas.height);

          // Scan for the gap in the background image by looking for
          // pixel value discontinuities (the gap is usually a transparent/white region)
          const w = bgCanvas.width;
          const h = bgCanvas.height;
          const scanY = Math.floor(h * 0.5); // scan middle row

          // Detect edges by measuring pixel differences
          let gapX = -1;
          let maxDiff = 0;
          const threshold = 30;

          for (let x = 10; x < w - 10; x++) {
            const idx1 = (scanY * w + x) * 4;
            const idx2 = (scanY * w + (x + 1)) * 4;
            const diff = Math.abs(bgData.data[idx1] - bgData.data[idx2]) +
                        Math.abs(bgData.data[idx1+1] - bgData.data[idx2+1]) +
                        Math.abs(bgData.data[idx1+2] - bgData.data[idx2+2]);

            if (diff > maxDiff) {
              maxDiff = diff;
              gapX = x;
            }
          }

          if (gapX > 0) {
            // Also check a few rows to confirm
            let confirmCount = 0;
            for (let y = Math.floor(h*0.3); y < Math.floor(h*0.7); y += 4) {
              const idx = (y * w + gapX) * 4;
              const idx2 = (y * w + (gapX + 1)) * 4;
              const d = Math.abs(bgData.data[idx] - bgData.data[idx2]) +
                       Math.abs(bgData.data[idx+1] - bgData.data[idx2+1]) +
                       Math.abs(bgData.data[idx+2] - bgData.data[idx2+2]);
              if (d > 20) confirmCount++;
            }

            if (confirmCount >= 3) {
              // gapX is the position in canvas pixels — map to viewport pixels
              const scaleX = bgCanvas.getBoundingClientRect().width / bgCanvas.width;
              return JSON.stringify({method:'canvas_gap', distance:Math.round(gapX * scaleX), gapX, canvasW:w, scaleX});
            }
          }
        } catch(e) { /* canvas may be tainted by CORS */ }
      }
    }

    // Method 2: Look for track element and calculate distance
    const trackSelectors = [
      '.gt_slider_knob', '.nc_scale', '.slider-track', '.slider-bg',
      '[class*="slider-track"]', '[class*="slide-track"]', '.slide-bar',
      '.yidun_slider__track', '.dx_captcha_slider_bg',
    ];
    for (const sel of trackSelectors) {
      const track = document.querySelector(sel);
      if (track) {
        const r = track.getBoundingClientRect();
        // Find knob inside or adjacent to track
        const parent = track.parentElement || track;
        const knob = parent.querySelector('[class*="knob"], [class*="btn"], [class*="button"], [class*="slider"] > div:not([class*="track"])');
        if (knob) {
          const kr = knob.getBoundingClientRect();
          return JSON.stringify({method:'track_width', distance:Math.round(r.width - kr.width)});
        }
        return JSON.stringify({method:'track_width_only', distance:Math.round(r.width * 0.85)});
      }
    }

    return JSON.stringify({method:'none', distance:0});
  })`, 15000);

  const distInfo = JSON.parse(distanceResult || '{"distance":0}');
  if (!distInfo.distance || distInfo.distance < 10) {
    if (debug) process.stderr.write(`[anti-bot][slider] no valid distance calculated\n`);
    return { passed: false };
  }

  if (debug) process.stderr.write(`[anti-bot][slider] distance=${distInfo.distance} method=${distInfo.method}\n`);

  // Now simulate the drag using CDP Input.dispatchMouseEvent
  // We need to send raw CDP commands through the proxy's /eval endpoint
  const dragResult = await simulateDrag(proxy, target, knob, distInfo.distance, debug);
  return dragResult;
}

// ---------------------------------------------------------------------------
// Strategy 5: CNKI-specific slider CAPTCHA
// CNKI uses a specific slider verification — optimized handler
// ---------------------------------------------------------------------------

async function tryCNKISlider(proxy, target, snap, debug) {
  // CNKI slider: look for the specific verification elements
  const detection = await evalJs(proxy, target, `(()=>{
    // CNKI commonly uses these patterns for their slider verification
    const sliders = [...document.querySelectorAll(
      '.yidun_slider, .yidun_slider_indicator, .yidun_slider__track, ' +
      '.verifybox, .slide-verify, .slideVerify, ' +
      '[id*="slide"], [class*="slide-verify"], [class*="SlideVerify"], ' +
      '.nc_wrapper, .nc_scale, .nc_iconfont, ' +
      '.captcha-block, .slider-verification, ' +
      '[class*="captcha"], [class*="verify"]'
    )].filter(el => {
      const r = el.getBoundingClientRect();
      return r.width > 0 && r.height > 0;
    });

    // Also look for canvas elements in verification context
    const canvases = [...document.querySelectorAll('canvas')].filter(c => {
      const r = c.getBoundingClientRect();
      return r.width > 50 && r.height > 50;
    });

    return JSON.stringify({
      sliderCount: sliders.length,
      canvasCount: canvases.length,
      hasVerifyText: /验证|滑块|滑动|拼图|拖动/i.test((document.body?.innerText||'') + (document.title||'')),
      sliderSelectors: sliders.slice(0,5).map(s => s.className?.toString().slice(0,60) || s.tagName),
      canvasSizes: canvases.map(c => ({w:c.width, h:c.height, rw:Math.round(c.getBoundingClientRect().width), rh:Math.round(c.getBoundingClientRect().height)})),
    });
  })`, 10000);

  const info = JSON.parse(detection || '{}');
  if (debug) process.stderr.write(`[anti-bot][cnki-slider] detection: ${JSON.stringify(info)}\n`);

  // If we found slider elements, try the generic slider handler
  if (info.sliderCount > 0 || (info.canvasCount >= 2 && info.hasVerifyText)) {
    return await trySliderCaptcha(proxy, target, snap, debug);
  }

  // CNKI sometimes shows a simple "click to verify" before the slider
  if (info.hasVerifyText && info.sliderCount === 0) {
    return await trySimpleClickChallenge(proxy, target, snap, debug);
  }

  return { passed: false };
}

// ---------------------------------------------------------------------------
// Strategy 6: reCAPTCHA Bypass — minimal attempt (mostly fails)
// ---------------------------------------------------------------------------

async function tryRecaptchaBypass(proxy, target, snap, debug) {
  // reCAPTCHA v2 (image grid) cannot be automatically solved.
  // reCAPTCHA v3 (invisible) is score-based and doesn't show a challenge.
  // We only attempt the simplest case: clicking the "I'm not a robot" checkbox
  // and hoping it auto-resolves (low-risk user profile, good IP reputation).
  const result = await evalJs(proxy, target, `(()=>{
    // Try to find and click reCAPTCHA checkbox
    const frames = [...document.querySelectorAll('iframe')];
    for (const frame of frames) {
      if (!/recaptcha|google\\.com\\/recaptcha/i.test(frame.src || '')) continue;
      try {
        const doc = frame.contentDocument || frame.contentWindow?.document;
        if (!doc) continue;
        const cb = doc.querySelector('.recaptcha-checkbox-border, .recaptcha-checkbox, #recaptcha-anchor, [role="checkbox"]');
        if (cb) {
          const rect = cb.getBoundingClientRect();
          // Simulate click at the center of the checkbox
          cb.dispatchEvent(new MouseEvent('click', {
            bubbles: true, cancelable: true,
            clientX: rect.left + rect.width/2,
            clientY: rect.top + rect.height/2,
          }));
          return JSON.stringify({clicked:true, method:'recaptcha_checkbox'});
        }
      } catch(e) { /* cross-origin — can't access iframe content */ }
    }
    return JSON.stringify({clicked:false});
  })`, 15000);

  const parsed = JSON.parse(result || '{"clicked":false}');
  if (!parsed.clicked) return { passed: false };

  // Wait for reCAPTCHA to potentially resolve
  await sleep(5000);
  await waitForComplete(proxy, target, 20000);

  // Check if reCAPTCHA widget shows resolved state
  const resolved = await evalJs(proxy, target, `(()=>{
    // Check for g-recaptcha-response
    const resp = document.querySelector('#g-recaptcha-response, [name="g-recaptcha-response"]');
    if (resp && resp.value && resp.value.length > 0) return true;
    // Check if the page has advanced (no longer shows verification)
    const txt = (document.title||'') + ' ' + (document.body?.innerText||'').slice(0,500);
    return !/captcha|robot|verify you are human|not a robot/i.test(txt);
  })`, 10000).catch(() => false);

  if (resolved) {
    return { passed: true, method: "recaptcha_auto" };
  }

  if (debug) process.stderr.write('[anti-bot][recaptcha] auto-resolve failed (expected — likely needs image challenge)\n');
  return { passed: false };
}

// ---------------------------------------------------------------------------
// Drag Simulation via CDP
// Uses raw CDP Input.dispatchMouseEvent to simulate human-like drag behavior
// ---------------------------------------------------------------------------

async function simulateDrag(proxy, target, knob, distance, debug) {
  const startX = knob.x + knob.w / 2;
  const startY = knob.y + knob.h / 2;
  const steps = 60 + Math.floor(Math.random() * 20); // human-like step count
  const noise = () => (Math.random() - 0.5) * 4; // Y-axis jitter

  if (debug) process.stderr.write(`[anti-bot][drag] start=(${startX},${startY}) distance=${distance} steps=${steps}\n`);

  // Helper: send a CDP command through the proxy's eval with a fake fetch
  async function dispatchMouse(type, x, y, button = "left", modifiers = 0) {
    const js = `fetch('http://localhost:0/cdp',{
      method:'POST',
      body:JSON.stringify({
        id:1,
        method:'Input.dispatchMouseEvent',
        params:{type:'${type}',x:${Math.round(x)},y:${Math.round(y)},button:'${button}',buttons:${type==='mouseReleased'?0:1},modifiers:${modifiers},clickCount:${type==='mousePressed'?1:0}}
      })
    }).catch(()=>{}); 1`;
    await evalJs(proxy, target, js, 5000).catch(() => {});
  }

  // Wait for the slider to be ready
  await sleep(300);

  // Phase 1: mousePressed + small initial move
  await dispatchMouse("mousePressed", startX, startY);
  await sleep(150 + Math.random() * 100);

  // Phase 2: Gradual drag with human-like acceleration/deceleration curve
  let currentX = 0;
  for (let i = 1; i <= steps; i++) {
    // Use ease-in-out curve: slow start, fast middle, slow end
    const progress = i / steps;
    const eased = progress < 0.5
      ? 2 * progress * progress
      : 1 - Math.pow(-2 * progress + 2, 2) / 2;

    const targetX = eased * distance;
    const stepX = startX + targetX;
    const stepY = startY + noise() * (i < steps * 0.1 || i > steps * 0.9 ? 0.5 : 2); // more jitter at start/end

    await dispatchMouse("mouseMoved", stepX, stepY);
    currentX = targetX;

    // Variable delay: slower at start and end
    const baseDelay = 8 + Math.random() * 15;
    const slowFactor = (progress < 0.15 || progress > 0.85) ? 3 : 1;
    await sleep(baseDelay * slowFactor);
  }

  // Small overshoot and correction (natural behavior)
  const overshoot = currentX + 3 + Math.random() * 5;
  await dispatchMouse("mouseMoved", startX + overshoot, startY + noise() * 2);
  await sleep(50 + Math.random() * 100);
  await dispatchMouse("mouseMoved", startX + distance, startY + noise() * 0.5);
  await sleep(150 + Math.random() * 100);

  // Phase 3: mouseReleased
  await dispatchMouse("mouseReleased", startX + distance, startY);

  // Wait for verification to complete
  await sleep(2000);

  // Check if verification passed
  const verifyResult = await evalJs(proxy, target, `(()=>{
    // Check common success indicators
    const txt = (document.title||'') + ' ' + (document.body?.innerText||'').slice(0,1000);
    const successMarkers = [
      /验证成功/i, /通过验证/i, /verification successful/i,
      /验证通过/i, /success/i,
    ];
    // Check if slider elements are gone (verification completed)
    const slidersGone = !document.querySelector('.yidun_slider, .gt_slider, .nc_wrapper, .slider-captcha, [class*="slider-verify"]');
    // Check if we're now on a search results or article page
    const isArticlePage = /article|detail|kcms|full-text|pdf|download|abstract|search results/i.test(txt);

    for (const re of successMarkers) {
      if (re.test(txt)) return JSON.stringify({passed:true, reason:'success_text'});
    }
    if (slidersGone && isArticlePage) return JSON.stringify({passed:true, reason:'slider_gone'});

    // Check for failure feedback
    const failMarkers = [/验证失败/i, /请重试/i, /请再试/i, /try again/i, /retry/i];
    for (const re of failMarkers) {
      if (re.test(txt)) return JSON.stringify({passed:false, reason:'fail_text'});
    }

    return JSON.stringify({passed:slidersGone, reason:slidersGone?'sliders_disappeared':'unknown'});
  })`, 10000);

  const vr = JSON.parse(verifyResult || '{"passed":false}');
  if (vr.passed) {
    return { passed: true, method: "slider_drag" };
  }

  if (debug) process.stderr.write(`[anti-bot][drag] verification not passed: ${vr.reason}\n`);

  // Try a second attempt with slightly different parameters
  await sleep(1000);
  // Reset position by doing a quick click elsewhere
  await dispatchMouse("mousePressed", startX - 50, startY);
  await dispatchMouse("mouseReleased", startX - 50, startY);
  await sleep(500);

  // Second attempt with different speed profile
  if (debug) process.stderr.write('[anti-bot][drag] retrying with different profile...\n');
  await dispatchMouse("mousePressed", startX, startY);
  await sleep(100);

  for (let i = 1; i <= steps; i++) {
    const progress = i / steps;
    // Linear-ish with slight randomness
    const eased = progress + (Math.random() - 0.5) * 0.02;
    const targetX = Math.min(distance, eased * distance);
    const stepX = startX + targetX;
    const stepY = startY + noise() * 1.5;
    await dispatchMouse("mouseMoved", stepX, stepY);
    await sleep(5 + Math.random() * 12);
  }

  await dispatchMouse("mouseReleased", startX + distance, startY);
  await sleep(2000);

  // Final check
  const finalCheck = await evalJs(proxy, target, `(()=>{
    const slidersGone = !document.querySelector('.yidun_slider, .gt_slider, .nc_wrapper, .slider-captcha, [class*="slider-verify"]');
    const txt = (document.title||'') + ' ' + (document.body?.innerText||'').slice(0,1000);
    const isArticlePage = /article|detail|kcms|full-text|pdf|download|abstract|搜索|search/i.test(txt);
    return JSON.stringify({passed: slidersGone || isArticlePage, slidersGone, isArticlePage});
  })`, 10000);

  const fc = JSON.parse(finalCheck || '{"passed":false}');
  return { passed: fc.passed, method: "slider_drag" };
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

async function pageSnapshot(proxy, target) {
  const info = await proxyGet(proxy, "/info", { target }, 10000).catch(() => ({}));
  const body = await evalJs(
    proxy,
    target,
    `(document.body && document.body.innerText || "").slice(0, 1500)`
  ).catch(() => "");
  return { url: info.url || "", title: info.title || "", body: body || "" };
}

// Re-export STATUS for convenience
export { STATUS };

// Shared CDP proxy client for nature-downloader scripts.
//
// All functions take the proxy URL as an explicit parameter (fixes the
// --proxy bug in batch_download.mjs where PROXY was a hard-coded const and
// args.proxy was parsed but never used).
//
// Endpoint reference (web-access CDP proxy v2.5.3+):
//   GET  /targets              -> [{targetId, url, title, ...}]
//   GET  /info?target=<id>     -> {targetId, url, title, ready, ...}
//   POST /new       (body=url) -> {targetId, ...}
//   POST /navigate  (body=url, ?target=id) -> {...}
//   POST /eval      (body=js,  ?target=id) -> {value, ...}
//   POST /click     (body=sel, ?target=id) -> {...}
//   GET  /close?target=<id>    -> {...}
//   GET  /scroll?target=<id>&direction=bottom -> {...}

import { STATUS } from "./status-codes.mjs";

export const DEFAULT_PROXY = "http://127.0.0.1:3456";

export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

/**
 * Check that the CDP proxy is reachable. Throws a friendly error with
 * remediation hints if not.
 */
export async function healthCheck(proxy = DEFAULT_PROXY, timeoutMs = 5000) {
  const u = new URL("/targets", proxy);
  let err;
  try {
    const r = await fetch(u, { signal: AbortSignal.timeout(timeoutMs) });
    if (!r.ok) {
      err = `HTTP ${r.status}`;
    } else {
      await r.json(); // parse to confirm it's the proxy, not a random page
      return true;
    }
  } catch (e) {
    err = e.code || e.name || String(e).split("\n")[0].slice(0, 80);
  }
  throw new Error(
    `CDP proxy not reachable at ${proxy} (${err}).\n` +
      `  Start: node <web-access>/scripts/check-deps.mjs\n` +
      `  Verify: chrome://inspect/#remote-debugging -> enable "Allow remote debugging"\n` +
      `  Probe: curl ${proxy}/targets`
  );
}

/**
 * Fetch JSON from a URL with timeout. Throws on HTTP error with status + body slice.
 */
export async function httpJson(url, options = {}) {
  const response = await fetch(url, {
    ...options,
    signal: AbortSignal.timeout(options.timeoutMs || 60000),
  });
  const text = await response.text();
  if (!response.ok) {
    throw new Error(`HTTP ${response.status} from ${url}: ${text.slice(0, 500)}`);
  }
  return JSON.parse(text);
}

/**
 * GET to a proxy endpoint with query params.
 */
export async function proxyGet(proxy, endpoint, params = {}, timeoutMs = 60000) {
  const u = new URL(endpoint, proxy);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  return httpJson(u.toString(), { timeoutMs });
}

/**
 * POST to a proxy endpoint where the URL/payload goes in the body
 * (web-access CDP proxy v2.5.3+: /new and /navigate take URL in POST body
 * so query params with their own ? fragments don't get mis-split).
 */
export async function proxyPostUrl(proxy, endpoint, url, params = {}, timeoutMs = 60000) {
  const u = new URL(endpoint, proxy);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  return httpJson(u.toString(), { method: "POST", body: url, timeoutMs });
}

/**
 * POST JS to /eval and return the full response object {value, ...}.
 */
export async function proxyEval(proxy, target, js, timeoutMs = 60000) {
  return proxyPostUrl(proxy, "/eval", js, { target }, timeoutMs);
}

/**
 * POST JS to /eval and return just the .value (the common case).
 */
export async function evalJs(proxy, target, js, timeoutMs = 60000) {
  const r = await proxyEval(proxy, target, js, timeoutMs);
  return r && r.value;
}

/**
 * Poll /info until ready === "complete" or maxMs elapsed.
 * Returns the last info object (may be null on persistent failure).
 */
export async function waitForComplete(proxy, target, maxMs = 45000) {
  const started = Date.now();
  let last = null;
  while (Date.now() - started < maxMs) {
    try {
      last = await proxyGet(proxy, "/info", { target }, 10000);
      if (last.ready === "complete") return last;
    } catch (_) {}
    await sleep(1000);
  }
  return last;
}

/**
 * Open a new tab at the given URL. Returns {targetId, ...}.
 */
export async function newTab(proxy, url, timeoutMs = 60000) {
  return proxyPostUrl(proxy, "/new", url, {}, timeoutMs);
}

/**
 * Navigate an existing tab to a URL.
 */
export async function navigate(proxy, target, url, timeoutMs = 60000) {
  return proxyPostUrl(proxy, "/navigate", url, { target }, timeoutMs);
}

/**
 * Close a tab. Swallows errors (best-effort cleanup).
 */
export async function closeTab(proxy, target) {
  try {
    await proxyGet(proxy, "/close", { target }, 10000);
  } catch (_) {}
}

/**
 * List all open targets/tabs.
 */
export async function listTargets(proxy) {
  return proxyGet(proxy, "/targets");
}

/**
 * Click an element matching a CSS selector in the target tab.
 * Swallows errors (used for cookie consent, search button — non-critical).
 */
export async function click(proxy, target, selector, timeoutMs = 30000) {
  try {
    return await proxyPostUrl(proxy, "/click", selector, { target }, timeoutMs);
  } catch (_) {
    return null;
  }
}

/**
 * Scroll the target tab in a direction (default "bottom").
 * Swallows errors.
 */
export async function scroll(proxy, target, direction = "bottom") {
  try {
    return await proxyGet(proxy, "/scroll", { target, direction }, 30000);
  } catch (_) {
    return null;
  }
}

import path from "node:path";
import { STATUS, classifyWall } from "./status-codes.mjs";
import {
  closeTab,
  evalJs,
  navigate,
  newTab,
  proxyGet,
  sleep,
  waitForComplete,
} from "./cdp-utils.mjs";
import { fetchAnyToFile, fetchNativeToFile, fetchToFile } from "./pdf-utils.mjs";
import { handleVerification } from "./anti-bot.mjs";

export const DEFAULT_CNKI_URL = "https://kns.cnki.net/kns8s/defaultresult/index";

const CNKI_HOST_RE = /(^|\.)cnki\.net$|(^|\.)cnki\.com\.cn$/i;
const CHINESE_RE = /[\u3400-\u9fff]/;

export function looksChinese(text = "") {
  return CHINESE_RE.test(String(text));
}

export function cnkiSearchUrl(query, baseUrl = DEFAULT_CNKI_URL) {
  const url = new URL(baseUrl);
  url.searchParams.set("kw", query);
  return url.toString();
}

export function safeCnkiFileName(title = "", ext = ".pdf") {
  const cleaned = String(title || "cnki-paper")
    .trim()
    .replace(/[\/:*?"<>|]+/g, "")
    .replace(/\s+/g, "_")
    .slice(0, 120);
  return `${cleaned || "cnki-paper"}${ext}`;
}

export function isCnkiUrl(url = "") {
  try {
    return CNKI_HOST_RE.test(new URL(url).hostname);
  } catch {
    return false;
  }
}

export function createCnkiTransport({ mode = "browser", entryUrl = DEFAULT_CNKI_URL, webvpnUrlTemplate = "" } = {}) {
  if (!["browser", "fsso", "webvpn"].includes(mode)) throw new Error(`unsupported CNKI transport mode: ${mode}`);
  return {
    mode,
    entryUrl,
    resolve(url) {
      if (mode !== "webvpn") return url;
      if (!webvpnUrlTemplate || !webvpnUrlTemplate.includes("{url}")) throw new Error("CNKI WebVPN transport requires webvpnUrlTemplate containing {url}");
      return webvpnUrlTemplate.replace("{url}", encodeURIComponent(url));
    },
  };
}

export function legacyCnkiDetailUrl({ filename = "", dbcode = "" } = {}) {
  if (!filename) return "";
  const url = new URL("https://kns.cnki.net/kcms/detail/detail.aspx");
  url.searchParams.set("filename", filename);
  if (dbcode) url.searchParams.set("dbcode", dbcode);
  return url.toString();
}

function classifyCnkiWall(url = "", title = "", body = "") {
  const wall = classifyWall(url, title, body);
  if (wall) return wall;
  const text = `${title} ${body}`;
  // Slider/drag/puzzle CAPTCHA — auto-attemptable
  if (/滑块|滑动验证|拖动滑块|拼图验证|请按住滑块|请拖动|slide to verify|drag the slider|slide verification|slider captcha/i.test(text)) {
    return { status: STATUS.PUBLISHER_VERIFICATION_WAITING_USER, reason: "CNKI slider captcha — auto-attemptable" };
  }
  // Login/auth wall
  if (/登录|统一身份认证|机构登录|校外访问|账号登录|扫码登录|验证码|安全验证|人机验证/i.test(text)) {
    return { status: STATUS.CARSI_WAITING_USER, reason: "CNKI or institutional login required" };
  }
  if (/没有权限|无权访问|未订购|未购买|余额不足|下载权限|未开通|403|forbidden/i.test(text)) {
    return { status: STATUS.LIBRARY_NO_PERMISSION, reason: "CNKI access denied" };
  }
  return null;
}

async function pageSnapshot(proxy, target) {
  const info = await proxyGet(proxy, "/info", { target }, 10000).catch(() => ({}));
  const body = await evalJs(
    proxy,
    target,
    `(document.body && document.body.innerText || "").slice(0,1000)`
  ).catch(() => "");
  return { url: info.url || "", title: info.title || "", body: body || "" };
}

async function submitSearchIfNeeded(proxy, target, query) {
  await evalJs(
    proxy,
    target,
    `(()=>{
      const value=${JSON.stringify(query)};
      const inputs=[...document.querySelectorAll('input[type="text"],input:not([type]),textarea')];
      const input=inputs.find(i=>/主题|篇名|关键词|检索|search|keyword|kw/i.test([i.placeholder,i.name,i.id,i.className].join(" ")))||inputs[0];
      if(!input)return false;
      const setter=Object.getOwnPropertyDescriptor(Object.getPrototypeOf(input),'value')?.set;
      if(setter)setter.call(input,value);else input.value=value;
      input.dispatchEvent(new Event('input',{bubbles:true}));
      input.dispatchEvent(new Event('change',{bubbles:true}));
      const buttons=[...document.querySelectorAll('button,input[type="button"],input[type="submit"],a')];
      const button=document.querySelector('input.search-btn,.search-btn')||
        buttons.find(e=>/检索|搜索|查询|Search/i.test(e.innerText||e.value||e.title||""));
      if(button){button.click();return true;}
      input.dispatchEvent(new KeyboardEvent('keydown',{key:'Enter',code:'Enter',bubbles:true}));
      return true;
    })()`,
    30000
  ).catch(() => false);
}

async function findResultUrl(proxy, target, title) {
  const raw = await evalJs(
    proxy,
    target,
    `(()=>{
      const norm=s=>String(s||'').normalize('NFKC').replace(/[\\s\\p{P}\\p{S}]+/gu,'').toLowerCase();
      const expected=norm(${JSON.stringify(title)});
      const links=[...document.querySelectorAll('a[href]')].map(a=>({
        href:a.href,
        text:(a.innerText||a.title||"").replace(/\\s+/g," ").trim(),
        filename:new URL(a.href,location.href).searchParams.get('filename')||new URL(a.href,location.href).searchParams.get('FileName')||'',
        dbcode:new URL(a.href,location.href).searchParams.get('dbcode')||''
      })).filter(x=>x.href&&/cnki\\.net|cnki\\.com\\.cn/i.test(x.href));
      const detail=links.filter(x=>/detail|KCMS|kcms|kns8s\\/Detail|dbcode|filename|FileName/i.test(x.href));
      const exact=detail.filter(x=>norm(x.text)===expected);
      return JSON.stringify(exact.length===1?exact[0]:null);
    })()`,
    30000
  );
  return JSON.parse(raw || "null");
}

async function findDownloadCandidates(proxy, target) {
  const raw = await evalJs(
    proxy,
    target,
    `(()=>{
      const out=[];
      const push=(url,text)=>{if(url&&/cnki\\.net|cnki\\.com\\.cn/i.test(url))out.push({url,text:text||""});};
      document.querySelectorAll('a[href],button,[onclick]').forEach(e=>{
        const text=(e.innerText||e.value||e.title||e.getAttribute('aria-label')||"").trim();
        const href=e.href||"";
        const onclick=e.getAttribute('onclick')||"";
        if(/PDF|整本下载|全文下载|下载|CAJ|HTML阅读|在线阅读/i.test(text+href+onclick)){
          push(href,text);
          const m=onclick.match(/https?:\\/\\/[^'"\\s)]+/i);
          if(m)push(m[0],text);
        }
      });
      return JSON.stringify([...new Map(out.map(x=>[x.url,x])).values()].slice(0,12));
    })()`,
    30000
  );
  const candidates = JSON.parse(raw || "[]");
  const score = (item) => {
    const s = `${item.text} ${item.url}`;
    if (/pdf/i.test(s)) return 0;
    if (/下载|download/i.test(s)) return 1;
    if (/caj/i.test(s)) return 2;
    return 3;
  };
  return candidates.sort((a, b) => score(a) - score(b));
}

async function downloadCnkiSupportingInformation(proxy, target, title, outDir) {
  const raw = await evalJs(
    proxy,
    target,
    `JSON.stringify([...new Set([...document.querySelectorAll('a[href]')].filter(a=>/补充材料|附件|supplementary|supporting information|附录/i.test((a.innerText||'')+' '+a.href)).map(a=>a.href))].slice(0,20))`
  ).catch(() => "[]");
  const links = JSON.parse(raw || "[]");
  const files = [];
  for (let index = 0; index < links.length; index += 1) {
    const url = links[index];
    const name = (new URL(url).pathname.split("/").pop() || `si-${index + 1}`).replace(/[\/:*?"<>|]+/g, "_");
    const outPath = path.join(outDir, "SupportingInformation", `${safeCnkiFileName(title, "")}__${name}`);
    const downloaded = await fetchAnyToFile(proxy, target, url, outPath, { rejectHtml: true }).catch(() => ({ ok: false }));
    if (downloaded.ok) files.push(downloaded.file);
  }
  return { status: files.length ? "downloaded" : "not_found", found: links.length, count: files.length, files };
}

export function filterCnkiDownloadCandidates(candidates = [], format = "any") {
  if (format !== "pdf") return candidates;
  return candidates.filter((candidate) => /pdf/i.test(`${candidate.text || ""} ${candidate.url || ""}`));
}

export async function downloadCnkiTitle(proxy, title, outDir, { cnkiUrl = DEFAULT_CNKI_URL, format = "any", debug = false, wantSi = false, transport = null } = {}) {
  const access = transport || createCnkiTransport({ entryUrl: cnkiUrl });
  const tab = (await newTab(proxy, access.resolve(cnkiSearchUrl(title, access.entryUrl)))).targetId;
  try {
    await waitForComplete(proxy, tab);
    await sleep(1500);
    await submitSearchIfNeeded(proxy, tab, title);
    await sleep(2500);
    await waitForComplete(proxy, tab);

    let snap = await pageSnapshot(proxy, tab);
    let wall = classifyCnkiWall(snap.url, snap.title, snap.body);
    if (wall) {
      if (debug) process.stderr.write(`[debug][cnki] wall after search: ${wall.status} "${wall.reason}" — attempting auto-verification...\n`);
      const vr = await handleVerification(proxy, tab, wall, { debug, maxAttempts: 2 });
      if (vr.passed) {
        if (debug) process.stderr.write(`[debug][cnki] auto-verification passed (${vr.method}), re-reading page...\n`);
        await sleep(1500);
        await waitForComplete(proxy, tab);
        snap = await pageSnapshot(proxy, tab);
        wall = classifyCnkiWall(snap.url, snap.title, snap.body);
        if (wall) {
          return { title, status: STATUS.VERIFICATION_AUTO_FAILED, url: snap.url, reason: `auto-verify failed (${vr.method}), still: ${wall.reason}` };
        }
      } else {
        return {
          title,
          status: vr.attempted ? STATUS.VERIFICATION_AUTO_FAILED : wall.status,
          url: snap.url,
          reason: vr.attempted ? `automatic verification did not resolve: ${wall.reason}` : wall.reason,
        };
      }
    }

    const hit = await findResultUrl(proxy, tab, title);
    if (!hit || !hit.href) {
      if (debug) process.stderr.write(`[debug][cnki] no result. url=${snap.url} title=${snap.title}\n`);
      return { title, status: STATUS.METADATA_AMBIGUOUS, url: snap.url, reason: "no unique exact CNKI title match" };
    }

    await navigate(proxy, tab, hit.href);
    await waitForComplete(proxy, tab);
    await sleep(1500);
    snap = await pageSnapshot(proxy, tab);
    if (/404|页面不存在|not found/i.test(`${snap.title} ${snap.body}`)) {
      const legacyUrl = legacyCnkiDetailUrl(hit);
      if (legacyUrl) {
        await navigate(proxy, tab, access.resolve(legacyUrl));
        await waitForComplete(proxy, tab);
        await sleep(1000);
        snap = await pageSnapshot(proxy, tab);
      }
    }
    wall = classifyCnkiWall(snap.url, snap.title, snap.body);
    if (wall) {
      if (debug) process.stderr.write(`[debug][cnki] wall after detail nav: ${wall.status} "${wall.reason}" — attempting auto-verification...\n`);
      const vr = await handleVerification(proxy, tab, wall, { debug, maxAttempts: 2 });
      if (vr.passed) {
        if (debug) process.stderr.write(`[debug][cnki] auto-verification passed (${vr.method}), re-reading page...\n`);
        await sleep(1500);
        await waitForComplete(proxy, tab);
        snap = await pageSnapshot(proxy, tab);
        wall = classifyCnkiWall(snap.url, snap.title, snap.body);
        if (wall) {
          return { title, status: STATUS.VERIFICATION_AUTO_FAILED, url: snap.url, reason: `auto-verify failed (${vr.method}), still: ${wall.reason}` };
        }
      } else {
        return {
          title,
          status: vr.attempted ? STATUS.VERIFICATION_AUTO_FAILED : wall.status,
          url: snap.url,
          reason: vr.attempted ? `automatic verification did not resolve: ${wall.reason}` : wall.reason,
        };
      }
    }

    const candidates = filterCnkiDownloadCandidates(await findDownloadCandidates(proxy, tab), format);
    if (!candidates.length) {
      if (debug) process.stderr.write(`[debug][cnki] no download candidates. url=${snap.url} title=${snap.title}\n`);
      return { title, status: STATUS.NO_AUTHORIZED_PDF_FOUND, url: snap.url, via: hit.href };
    }

    for (const candidate of candidates) {
      const lower = `${candidate.text} ${candidate.url}`.toLowerCase();
      const ext = lower.includes("caj") ? ".caj" : ".pdf";
      const outPath = path.join(outDir, ext === ".caj" ? "CNKI" : "PDFs", safeCnkiFileName(title, ext));
      const got = ext === ".pdf"
        ? await fetchToFile(proxy, tab, candidate.url, outPath).catch((e) => ({ ok: false, err: String(e).slice(0, 120) }))
        : await fetchNativeToFile(proxy, tab, candidate.url, outPath, { allowedFormats: ["caj"] }).catch((e) => ({ ok: false, err: String(e).slice(0, 120) }));
      if (got.ok) {
        const result = {
          title,
          status: ext === ".pdf" ? STATUS.DOWNLOADED : STATUS.NATIVE_FULLTEXT_DOWNLOADED,
          file: got.file,
          bytes: got.bytes,
          via: snap.url,
          source: "cnki",
          format: ext.slice(1),
          accessMode: "institution_browser",
          transport: access.mode,
        };
        if (wantSi) {
          result.si = await downloadCnkiSupportingInformation(proxy, tab, title, outDir);
          if ((result.si?.count || 0) > 0) result.status = STATUS.DOWNLOADED_WITH_SI;
        }
        return result;
      }
    }

    return { title, status: STATUS.PDF_FETCH_FAILED, url: snap.url, via: hit.href, source: "cnki" };
  } finally {
    await closeTab(proxy, tab);
  }
}

export async function downloadCnkiDirectUrl(proxy, url, title, outDir, { debug = false, wantSi = false } = {}) {
  const tab = (await newTab(proxy, url)).targetId;
  try {
    await waitForComplete(proxy, tab);
    const snapshot = await pageSnapshot(proxy, tab);
    const wall = classifyCnkiWall(snapshot.url, snapshot.title, snapshot.body);
    if (wall) return { title, status: wall.status, url: snapshot.url, reason: wall.reason, source: "cnki" };
    const ext = /caj/i.test(`${url} ${snapshot.url}`) ? ".caj" : ".pdf";
    const file = path.join(outDir, ext === ".pdf" ? "PDFs" : "CNKI", safeCnkiFileName(title || "cnki-paper", ext));
    const downloaded = ext === ".pdf"
      ? await fetchToFile(proxy, tab, url, file)
      : await fetchNativeToFile(proxy, tab, url, file, { allowedFormats: ["caj"] });
    if (!downloaded.ok) return { title, status: STATUS.PDF_FETCH_FAILED, url, err: downloaded.err, source: "cnki" };
    const result = {
      title,
      status: ext === ".pdf" ? STATUS.DOWNLOADED : STATUS.NATIVE_FULLTEXT_DOWNLOADED,
      file: downloaded.file,
      bytes: downloaded.bytes,
      format: ext.slice(1),
      source: "cnki",
      accessMode: "institution_browser",
    };
    if (wantSi) {
      result.si = await downloadCnkiSupportingInformation(proxy, tab, title, outDir);
      if ((result.si?.count || 0) > 0) result.status = STATUS.DOWNLOADED_WITH_SI;
    }
    return result;
  } catch (error) {
    if (debug) process.stderr.write(`[debug][cnki] direct URL failed: ${String(error)}\n`);
    return { title, status: STATUS.FAILED_AFTER_RETRY, url, err: String(error).slice(0, 160), source: "cnki" };
  } finally {
    await closeTab(proxy, tab);
  }
}

import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { PUBLISHER_PROVIDERS } from "./provider-registry.mjs";

const PROVIDERS = new Set(PUBLISHER_PROVIDERS);

export function credentialsPathFromEnv(env = process.env) {
  const dir = env.LIT_DL_CONFIG_DIR || path.join(os.homedir(), ".config", "lit-dl");
  return path.join(dir, "credentials.json");
}

export function loadCredentials(env = process.env) {
  const file = credentialsPathFromEnv(env);
  if (!fs.existsSync(file)) return {};
  try {
    const value = JSON.parse(fs.readFileSync(file, "utf8"));
    return value && typeof value === "object" ? value : {};
  } catch {
    return {};
  }
}

function assertProvider(provider) {
  if (!PROVIDERS.has(provider)) throw new Error(`unsupported provider: ${provider}`);
}

function writeCredentials(value, env) {
  const file = credentialsPathFromEnv(env);
  fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
  const temp = `${file}.tmp-${process.pid}`;
  fs.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
  fs.chmodSync(temp, 0o600);
  fs.renameSync(temp, file);
  fs.chmodSync(file, 0o600);
  return file;
}

export function saveProviderCredentials(provider, secrets, env = process.env) {
  assertProvider(provider);
  const clean = Object.fromEntries(
    Object.entries(secrets || {}).filter(([, value]) => typeof value === "string" && value.trim()).map(([key, value]) => [key, value.trim()])
  );
  if (!clean.api_key) throw new Error("api_key is required");
  const all = loadCredentials(env);
  all[provider] = clean;
  writeCredentials(all, env);
  return clean;
}

export function deleteProviderCredentials(provider, env = process.env) {
  assertProvider(provider);
  const all = loadCredentials(env);
  delete all[provider];
  writeCredentials(all, env);
}

export function providerCredentials(provider, env = process.env) {
  assertProvider(provider);
  return loadCredentials(env)[provider] || null;
}

export function maskSecret(value = "") {
  const text = String(value);
  if (!text) return "";
  if (text.length <= 4) return "*".repeat(text.length);
  return `${"*".repeat(text.length - 4)}${text.slice(-4)}`;
}

export function maskedCredentials(env = process.env) {
  const all = loadCredentials(env);
  return Object.fromEntries(Object.entries(all).map(([provider, values]) => [
    provider,
    Object.fromEntries(Object.entries(values || {}).map(([key, value]) => [key, maskSecret(value)])),
  ]));
}

import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { classifyFullTextContent } from "./provider-utils.mjs";

export function safeArticleBasename(title = "", doi = "") {
  const base = String(title || doi || "article")
    .trim()
    .replace(/[\/:*?"<>|]+/g, "")
    .replace(/\s+/g, "_")
    .slice(0, 140);
  return base || "article";
}

export async function saveFullTextResponse(response, { outDir, title = "", doi = "", source = "" } = {}) {
  if (!response.ok) return { ok: false, httpStatus: response.status };
  const body = Buffer.from(await response.arrayBuffer());
  const contentType = response.headers.get("content-type") || "";
  const classification = classifyFullTextContent({ contentType, head: body.subarray(0, 65536) });
  if (!classification.valid) return { ok: false, httpStatus: response.status, contentType, ...classification };
  const folder = classification.format === "pdf" ? "PDFs" : "FullText";
  const file = path.join(outDir, folder, `${safeArticleBasename(title, doi)}${classification.extension}`);
  fs.mkdirSync(path.dirname(file), { recursive: true });
  const temp = `${file}.part-${process.pid}`;
  fs.writeFileSync(temp, body);
  fs.renameSync(temp, file);
  return {
    ok: true,
    file,
    bytes: body.length,
    sha256: crypto.createHash("sha256").update(body).digest("hex"),
    contentType,
    format: classification.format,
    source: source || response.url || "",
  };
}

import fs from "node:fs";
import path from "node:path";

const SECRET_KEY_RE = /(?:api[_-]?key|secret|password|cookie|session|authtoken|insttoken|access[_-]?token|authorization)/i;
const SECRET_QUERY_RE = /([?&](?:api[_-]?key|apikey|secret|password|authtoken|insttoken|access[_-]?token|authorization)=)[^&#\s"']*/gi;

function redactString(value) {
  return value
    .replace(SECRET_QUERY_RE, "$1[REDACTED]")
    .replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[REDACTED]");
}

function redact(value) {
  if (Array.isArray(value)) return value.map(redact);
  if (typeof value === "string") return redactString(value);
  if (!value || typeof value !== "object") return value;
  return Object.fromEntries(
    Object.entries(value)
      .filter(([key]) => !SECRET_KEY_RE.test(key))
      .map(([key, child]) => [key, redact(child)])
  );
}

export function writeManifest(outDir, manifest) {
  fs.mkdirSync(outDir, { recursive: true });
  const file = path.join(outDir, "manifest.json");
  const temp = `${file}.tmp-${process.pid}`;
  const payload = {
    version: 2,
    generated_at: new Date().toISOString(),
    ...redact(manifest),
  };
  fs.writeFileSync(temp, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
  fs.renameSync(temp, file);
  return file;
}

export function normalizeDoi(value = "") {
  return String(value).trim().replace(/^https?:\/\/(?:dx\.)?doi\.org\//i, "").toLowerCase();
}

export function parseCrossrefWork(work = {}) {
  const links = Array.isArray(work.link) ? work.link : [];
  const pdfLink = links.find((link) => /application\/pdf/i.test(link?.["content-type"] || ""));
  return {
    doi: normalizeDoi(work.DOI || work.doi || ""),
    title: Array.isArray(work.title) ? work.title[0] || "" : work.title || "",
    publisher: work.publisher || "",
    language: work.language || "",
    license: Array.isArray(work.license) ? work.license[0]?.URL || "" : "",
    publisherPdfUrl: pdfLink?.URL || "",
    sourceUrl: work.URL || "",
    type: work.type || "",
  };
}

export async function fetchCrossrefByDoi(doi, { fetchImpl = fetch, mailto = "" } = {}) {
  const headers = { Accept: "application/json" };
  if (mailto) headers["User-Agent"] = `nature-downloader/1.0 (mailto:${mailto})`;
  const response = await fetchImpl(`https://api.crossref.org/works/${encodeURIComponent(normalizeDoi(doi))}`, {
    headers,
    signal: AbortSignal.timeout(30000),
  });
  if (!response.ok) throw new Error(`Crossref DOI lookup failed: HTTP ${response.status}`);
  const body = await response.json();
  return parseCrossrefWork(body?.message || {});
}

export async function findCrossrefByTitle(title, { fetchImpl = fetch, mailto = "" } = {}) {
  const url = new URL("https://api.crossref.org/works");
  url.searchParams.set("query.title", title);
  url.searchParams.set("rows", "5");
  if (mailto) url.searchParams.set("mailto", mailto);
  const response = await fetchImpl(url, { headers: { Accept: "application/json" }, signal: AbortSignal.timeout(30000) });
  if (!response.ok) throw new Error(`Crossref title lookup failed: HTTP ${response.status}`);
  const items = (await response.json())?.message?.items || [];
  const normalize = (value) => String(value || "").replace(/\s+/g, " ").trim().toLowerCase();
  const exact = items.find((item) => normalize(Array.isArray(item.title) ? item.title[0] : item.title) === normalize(title));
  return exact ? parseCrossrefWork(exact) : null;
}

import { saveFullTextResponse } from "./direct-download.mjs";
import {
  findPmcCandidates,
  fetchUnpaywallRecord,
  parseUnpaywallRecord,
  rankOaCandidates,
} from "./open-access.mjs";
import { STATUS } from "./status-codes.mjs";

function httpsForFtp(url = "") {
  return String(url).replace(/^ftp:\/\/ftp\.ncbi\.nlm\.nih\.gov\//i, "https://ftp.ncbi.nlm.nih.gov/");
}

async function tryCandidates(article, candidates, { outDir, fetchImpl }) {
  const attempts = [];
  for (const candidate of rankOaCandidates(candidates)) {
    const url = httpsForFtp(candidate.url);
    let response;
    try {
      response = await fetchImpl(url, { headers: { Accept: "application/pdf, text/html, application/xml" }, signal: AbortSignal.timeout(60000) });
    } catch (error) {
      attempts.push({ source: candidate.source, url, status: "request_failed", reason: String(error?.message || error).slice(0, 120) });
      continue;
    }
    const saved = await saveFullTextResponse(response, { outDir, title: article.title, doi: article.doi, source: url });
    if (!saved.ok) {
      attempts.push({ source: candidate.source, url, status: "invalid_or_unavailable", http_status: saved.httpStatus, reason: saved.reason || "invalid content" });
      continue;
    }
    return { downloaded: {
      status: saved.format === "html"
        ? STATUS.FULL_TEXT_HTML_AVAILABLE
        : saved.format === "pdf"
          ? STATUS.OPEN_ACCESS_DOWNLOADED
          : STATUS.NATIVE_FULLTEXT_DOWNLOADED,
      provider: "open_access",
      accessMode: "open_access",
      oaEvidence: candidate,
      ...saved,
      oaAttempts: attempts,
    }, attempts };
  }
  return { downloaded: null, attempts };
}

async function findPmcidByDoi(doi, fetchImpl) {
  if (!doi) return { pmcid: "", checked: false };
  const url = new URL("https://www.ebi.ac.uk/europepmc/webservices/rest/search");
  url.searchParams.set("query", `DOI:${doi}`);
  url.searchParams.set("format", "json");
  const response = await fetchImpl(url, { headers: { Accept: "application/json" }, signal: AbortSignal.timeout(30000) });
  if (!response.ok) return { pmcid: "", checked: false, error: `Europe PMC HTTP ${response.status}` };
  const record = (await response.json())?.resultList?.result?.[0];
  return { pmcid: record?.pmcid || "", checked: true };
}

export async function downloadOpenAccessArticle(article, {
  email = "",
  outDir,
  fetchImpl = fetch,
} = {}) {
  const pmcLookup = article.pmcid
    ? { pmcid: article.pmcid, checked: true }
    : await findPmcidByDoi(article.doi, fetchImpl).catch((error) => ({ pmcid: "", checked: false, error: String(error?.message || error).slice(0, 120) }));
  const pmcid = pmcLookup.pmcid;
  const allAttempts = [];
  if (pmcid) {
    const pmc = await findPmcCandidates(pmcid, { fetchImpl }).catch(() => []);
    const attempt = await tryCandidates(article, pmc, { outDir, fetchImpl });
    allAttempts.push(...attempt.attempts);
    if (attempt.downloaded) return attempt.downloaded;
  }

  let unpaywallRecord = null;
  let unpaywallChecked = false;
  let unpaywallError = "";
  if (article.doi && email) {
    try {
      unpaywallRecord = await fetchUnpaywallRecord(article.doi, email, { fetchImpl });
      unpaywallChecked = true;
    } catch (error) {
      unpaywallError = String(error?.message || error).slice(0, 120);
    }
  }
  const unpaywall = parseUnpaywallRecord(unpaywallRecord);
  const publisher = article.publisherPdfUrl && article.license
    ? [{ source: "publisher_oa", url: article.publisherPdfUrl, format: "pdf", version: "publishedVersion", license: article.license }]
    : [];
  const attempt = await tryCandidates(article, [...unpaywall, ...publisher], { outDir, fetchImpl });
  allAttempts.push(...attempt.attempts);
  if (attempt.downloaded) return attempt.downloaded;
  const confirmedOa = Boolean(pmcid || unpaywallRecord?.is_oa || article.license);
  const confirmedClosed = Boolean(unpaywallChecked && unpaywallRecord?.is_oa === false);
  return {
    status: STATUS.OA_NOT_FOUND,
    provider: "open_access",
    doi: article.doi,
    title: article.title,
    oaAssessment: confirmedOa ? "confirmed_oa" : confirmedClosed ? "confirmed_closed" : "unknown",
    oaChecks: {
      pmc: { checked: pmcLookup.checked, pmcid: pmcid || "", error: pmcLookup.error || "" },
      unpaywall: { checked: unpaywallChecked, error: unpaywallError, is_oa: unpaywallRecord?.is_oa ?? null },
      crossref_license: article.license || "",
    },
    oaAttempts: allAttempts,
  };
}

import path from "node:path";

export function normalizeTitle(title = "") {
  return String(title).replace(/\s+/g, " ").trim().toLowerCase();
}

export function exactTitleMatch(candidate, expected) {
  return normalizeTitle(candidate) === normalizeTitle(expected);
}

export function normalizeArxivId(value = "") {
  const text = String(value).trim();
  const match = text.match(/(?:arxiv\.org\/(?:abs|pdf)\/)?([a-z-]+\/\d{7}|\d{4}\.\d{4,5})(v\d+)?(?:\.pdf)?/i);
  if (!match) return "";
  return `${match[1]}${match[2] || ""}`;
}

export function arxivPdfUrl(id) {
  const normalized = normalizeArxivId(id);
  if (!normalized) return "";
  return `https://arxiv.org/pdf/${normalized}`;
}

function decodeXml(text = "") {
  return text
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&amp;/g, "&")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'");
}

export function parseArxivAtom(xml, expectedTitle) {
  const entries = String(xml).match(/<entry>[\s\S]*?<\/entry>/g) || [];
  for (const entry of entries) {
    const title = decodeXml((entry.match(/<title>([\s\S]*?)<\/title>/) || [])[1] || "").replace(/\s+/g, " ").trim();
    if (!exactTitleMatch(title, expectedTitle)) continue;
    const rawId = decodeXml((entry.match(/<id>([\s\S]*?)<\/id>/) || [])[1] || "");
    const id = normalizeArxivId(rawId);
    if (!id) continue;
    return { id, title, pdfUrl: arxivPdfUrl(id) };
  }
  return null;
}

export async function findArxivByTitle(title, { fetchImpl = fetch } = {}) {
  const url = new URL("https://export.arxiv.org/api/query");
  url.searchParams.set("search_query", `ti:"${title}"`);
  url.searchParams.set("start", "0");
  url.searchParams.set("max_results", "5");
  const response = await fetchImpl(url, { signal: AbortSignal.timeout(30000) });
  if (!response.ok) {
    throw new Error(`arXiv lookup failed: HTTP ${response.status}`);
  }
  return parseArxivAtom(await response.text(), title);
}

export function parsePmcOaXml(xml = "") {
  const links = [];
  const linkRe = /<link\b([^>]+)>/gi;
  for (const match of String(xml).matchAll(linkRe)) {
    const attrs = match[1];
    const format = (attrs.match(/\bformat=["']([^"']+)/i) || [])[1] || "";
    const href = (attrs.match(/\bhref=["']([^"']+)/i) || [])[1] || "";
    if (!href || !/pdf/i.test(format)) continue;
    links.push({ source: "pmc", url: href, format: "pdf", version: "publishedVersion", license: "pmc-oa" });
  }
  return links;
}

export function parseUnpaywallRecord(record = {}) {
  if (!record?.is_oa) return [];
  const locations = [record.best_oa_location, ...(record.oa_locations || [])].filter(Boolean);
  const seen = new Set();
  const out = [];
  for (const location of locations) {
    const url = location.url_for_pdf || location.url || "";
    if (!url || seen.has(url)) continue;
    seen.add(url);
    out.push({
      source: location.host_type === "publisher" ? "publisher_oa" : "repository",
      resolver: "unpaywall",
      url,
      format: location.url_for_pdf ? "pdf" : "landing",
      hostType: location.host_type || "",
      version: location.version || "",
      license: location.license || record.license || "",
    });
  }
  return out;
}

export function rankOaCandidates(candidates = []) {
  const sourceRank = { pmc: 0, unpaywall: 1, publisher_oa: 2, repository: 3, arxiv: 4 };
  return [...candidates].sort((a, b) => {
    const source = (sourceRank[a.source] ?? 99) - (sourceRank[b.source] ?? 99);
    if (source) return source;
    return (a.format === "pdf" ? 0 : 1) - (b.format === "pdf" ? 0 : 1);
  });
}

export async function findPmcCandidates(pmcid, { fetchImpl = fetch } = {}) {
  if (!pmcid) return [];
  const response = await fetchImpl(`https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi?id=${encodeURIComponent(pmcid)}`, { signal: AbortSignal.timeout(30000) });
  if (!response.ok) return [];
  return parsePmcOaXml(await response.text());
}

export async function findUnpaywallCandidates(doi, email, { fetchImpl = fetch } = {}) {
  if (!doi || !email) return [];
  const record = await fetchUnpaywallRecord(doi, email, { fetchImpl });
  return parseUnpaywallRecord(record);
}

export async function fetchUnpaywallRecord(doi, email, { fetchImpl = fetch } = {}) {
  if (!doi || !email) return null;
  const response = await fetchImpl(`https://api.unpaywall.org/v2/${encodeURIComponent(doi)}?email=${encodeURIComponent(email)}`, { signal: AbortSignal.timeout(30000) });
  if (!response.ok) throw new Error(`Unpaywall lookup failed: HTTP ${response.status}`);
  return response.json();
}

export function filenameForPdfUrl(url, title = "") {
  if (title) {
    const safeTitle = title
      .trim()
      .replace(/[\/:*?"<>|]+/g, "")
      .replace(/\s+/g, "_")
      .slice(0, 120);
    if (safeTitle) return `${safeTitle}.pdf`;
  }
  const base = path.basename(new URL(url).pathname) || "paper.pdf";
  return base.endsWith(".pdf") ? base : `${base}.pdf`;
}

// PDF fetch + disk-streaming helpers for nature-downloader.
//
// All functions take proxy + target explicitly so they work regardless of
// which script calls them. Bytes are fetched inside the page's authenticated
// context via fetch(), then transferred to Node in base64 chunks and written
// to disk. This is the same approach as the original code but:
//  - deduplicated (fetchToFile / fetchAnyToFile share fetchToBuffer + streamToDisk)
//  - window variable is randomized + deleted after use (avoids multi-tab collisions)
//  - maxBytes guard prevents OOM on huge files
//  - requirePdf flag controls %PDF head validation

import fs from "node:fs";
import path from "node:path";
import { evalJs } from "./cdp-utils.mjs";
import { STATUS } from "./status-codes.mjs";
import { classifyFullTextContent } from "./provider-utils.mjs";

const DEFAULT_MAX_BYTES = 200 * 1024 * 1024; // 200 MB guard
const DEFAULT_CHUNK = 1048576; // 1 MB per base64 round-trip

/**
 * Check if a byte array starts with the %PDF signature.
 */
export function isPdfHead(bytes) {
  if (!bytes || bytes.length < 5) return false;
  const head = String.fromCharCode(...bytes.slice(0, 5));
  return head === "%PDF-";
}

export function isHtmlResponse({ contentType = "", head = [] } = {}) {
  if (/\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType)) return true;
  const prefix = Buffer.from(head || []).toString("utf8").trimStart().toLowerCase();
  return prefix.startsWith("<!doctype html") || prefix.startsWith("<html") || prefix.startsWith("<head");
}

export function shouldRejectHtmlResponse(meta, rejectHtml = false) {
  return Boolean(rejectHtml && isHtmlResponse(meta));
}

/**
 * Fetch a URL inside the target tab's authenticated context.
 * Returns { ok, status, size, head, contentType, url } or { ok:false, err }.
 *
 * The bytes are stored in a randomized window variable to avoid collisions
 * when multiple tabs download concurrently.
 */
export async function fetchToBuffer(
  proxy,
  target,
  url,
  { requirePdf = true, maxBytes = DEFAULT_MAX_BYTES } = {}
) {
  // Random window var name so concurrent tabs don't clobber each other.
  const varName = `__litDlBytes_${Math.random().toString(36).slice(2, 10)}`;
  const js = `(async()=>{try{
    const r=await fetch(${JSON.stringify(url)},{credentials:'include'});
    const ab=await r.arrayBuffer();
    const b=new Uint8Array(ab);
    if(b.length>${maxBytes}){return JSON.stringify({ok:false,err:'pdf_too_large',size:b.length});}
    window['${varName}']=b;
    return JSON.stringify({ok:r.ok,status:r.status,size:b.length,head:Array.from(b.slice(0,64)),contentType:r.headers.get('content-type')||'',contentDisposition:r.headers.get('content-disposition')||'',url:r.url||location.href});
  }catch(e){return JSON.stringify({ok:false,err:String(e).slice(0,200)})}})()`;
  const raw = await evalJs(proxy, target, js, 120000);
  const meta = JSON.parse(raw || "{}");

  if (!meta.ok || !meta.size) {
    return { ok: false, err: meta.err || "empty response", varName };
  }
  if (meta.err === "pdf_too_large") {
    return { ok: false, err: STATUS.PDF_TOO_LARGE, size: meta.size, varName };
  }
  if (requirePdf) {
    const headBytes = meta.head || [];
    if (!isPdfHead(headBytes)) {
      // Clean up the window var before returning.
      await evalJs(proxy, target, `delete window['${varName}']`).catch(() => {});
      return { ok: false, err: "not a PDF (head mismatch)", head: meta.head, varName };
    }
  }
  return {
    ok: true,
    status: meta.status,
    size: meta.size,
    head: meta.head,
    contentType: meta.contentType,
    contentDisposition: meta.contentDisposition,
    url: meta.url,
    varName,
  };
}

/**
 * Stream bytes from a window variable to disk in base64 chunks.
 * Deletes the window variable when done (or on error).
 */
export async function streamToDisk(
  proxy,
  target,
  varName,
  size,
  outPath,
  chunkSize = DEFAULT_CHUNK,
  onProgress
) {
  fs.mkdirSync(path.dirname(outPath), { recursive: true });
  const ws = fs.createWriteStream(outPath);
  try {
    for (let s = 0; s < size; s += chunkSize) {
      const e = Math.min(s + chunkSize, size);
      const b64 = await evalJs(
        proxy,
        target,
        `(()=>{const b=window['${varName}'].slice(${s},${e});let x='';for(let i=0;i<b.length;i+=0x8000){x+=String.fromCharCode.apply(null,b.subarray(i,i+0x8000));}return btoa(x);})()`,
        60000
      );
      ws.write(Buffer.from(b64, "base64"));
      if (onProgress) onProgress(e, size);
    }
    await new Promise((r) => ws.end(r));
  } finally {
    // Always clean up the window var, even on error.
    await evalJs(proxy, target, `delete window['${varName}']`).catch(() => {});
  }
  return { file: outPath, bytes: size };
}

/**
 * Fetch a URL (requiring %PDF) and stream to disk.
 * Returns { ok:true, file, bytes } or { ok:false, err }.
 */
export async function fetchToFile(proxy, target, url, outPath, { onProgress, maxBytes } = {}) {
  const meta = await fetchToBuffer(proxy, target, url, { requirePdf: true, maxBytes });
  if (!meta.ok) return { ok: false, err: meta.err };
  const res = await streamToDisk(
    proxy,
    target,
    meta.varName,
    meta.size,
    outPath,
    DEFAULT_CHUNK,
    onProgress
  );
  return { ok: true, ...res };
}

/**
 * Like fetchToFile but accepts any binary (SI can be jpg/xlsx/docx — not PDF).
 * Returns { ok:true, bytes } or { ok:false, err }.
 */
export async function fetchAnyToFile(proxy, target, url, outPath, { onProgress, maxBytes, rejectHtml = false } = {}) {
  const meta = await fetchToBuffer(proxy, target, url, { requirePdf: false, maxBytes });
  if (!meta.ok) return { ok: false, err: meta.err };
  if (shouldRejectHtmlResponse(meta, rejectHtml)) {
    await evalJs(proxy, target, `delete window['${meta.varName}']`).catch(() => {});
    return { ok: false, err: "HTML response rejected" };
  }
  const resolvedOutPath = typeof outPath === "function" ? outPath(meta) : outPath;
  const res = await streamToDisk(
    proxy,
    target,
    meta.varName,
    meta.size,
    resolvedOutPath,
    DEFAULT_CHUNK,
    onProgress
  );
  return {
    ok: true,
    file: res.file,
    bytes: res.bytes,
    contentType: meta.contentType,
    contentDisposition: meta.contentDisposition,
    finalUrl: meta.url,
  };
}

export async function fetchNativeToFile(proxy, target, url, outPath, { allowedFormats = ["caj", "html", "jats_xml"], onProgress, maxBytes } = {}) {
  const meta = await fetchToBuffer(proxy, target, url, { requirePdf: false, maxBytes });
  if (!meta.ok) return { ok: false, err: meta.err };
  const classification = classifyFullTextContent({ contentType: meta.contentType, head: meta.head });
  if (!classification.valid || !allowedFormats.includes(classification.format)) {
    await evalJs(proxy, target, `delete window['${meta.varName}']`).catch(() => {});
    return { ok: false, err: classification.reason || `unexpected format ${classification.format}` };
  }
  const result = await streamToDisk(proxy, target, meta.varName, meta.size, outPath, DEFAULT_CHUNK, onProgress);
  return { ok: true, ...result, format: classification.format, contentType: meta.contentType, finalUrl: meta.url };
}

import fs from "node:fs";

const file = new URL("../../data/publishers.json", import.meta.url);
export const PROVIDER_REGISTRY = Object.freeze(JSON.parse(fs.readFileSync(file, "utf8")));
export const PUBLISHER_PROVIDERS = Object.freeze(Object.keys(PROVIDER_REGISTRY));

export function providerDescriptor(provider) {
  return PROVIDER_REGISTRY[provider] || null;
}

import { STATUS } from "./status-codes.mjs";

const LOGIN_RE = /\b(?:log[ -]?in|sign[ -]?in|password|authentication required|access denied|forbidden)\b|登录|统一身份认证|无权访问/i;

function bytesToText(head) {
  return Buffer.from(head || []).toString("utf8");
}

export function classifyFullTextContent({ contentType = "", head = Buffer.alloc(0) } = {}) {
  const bytes = Buffer.from(head || []);
  const text = bytesToText(bytes).trimStart();
  const lowerType = String(contentType).toLowerCase();
  if (bytes.subarray(0, 5).toString("ascii") === "%PDF-") return { valid: true, format: "pdf", extension: ".pdf" };
  if (/^(?:CAJ|HN|KDH)/.test(bytes.subarray(0, 12).toString("ascii"))) return { valid: true, format: "caj", extension: ".caj" };
  if (/html|xhtml/.test(lowerType) || /^<!doctype html|^<html|^<head/i.test(text)) {
    if (LOGIN_RE.test(text)) return { valid: false, format: "html", reason: "login_or_error_page" };
    if (/<article[\s>]|\bfull[- ]text\b|class=["'][^"']*(?:article-body|fulltext)/i.test(text)) {
      return { valid: true, format: "html", extension: ".html" };
    }
    return { valid: false, format: "html", reason: "not_fulltext_html" };
  }
  if (/xml|jats/.test(lowerType) || /^<\?xml|^<article[\s>]/i.test(text)) {
    if (/<article[\s>]|<book-part-wrapper[\s>]/i.test(text)) return { valid: true, format: "jats_xml", extension: ".xml" };
    return { valid: false, format: "xml", reason: "not_jats_fulltext" };
  }
  return { valid: false, format: "unknown", reason: "unsupported_or_invalid_content" };
}

export function classifyProviderFailure({ status = 0 } = {}) {
  if (status === 401) return STATUS.CREDENTIALS_INVALID;
  if (status === 403) return STATUS.API_NOT_ENTITLED;
  if (status === 404 || status === 204) return STATUS.API_FULLTEXT_UNAVAILABLE;
  if (status === 429) return STATUS.RATE_LIMITED;
  return STATUS.API_FULLTEXT_UNAVAILABLE;
}

import { saveFullTextResponse } from "./direct-download.mjs";
import { classifyProviderFailure } from "./provider-utils.mjs";
import { STATUS } from "./status-codes.mjs";
import { PROVIDER_REGISTRY } from "./provider-registry.mjs";

export const PROVIDER_CONFIG_URLS = Object.freeze(Object.fromEntries(
  Object.entries(PROVIDER_REGISTRY).map(([provider, descriptor]) => [provider, descriptor.config_url])
));

const SECRET_QUERY_PARAM_RE = /^(?:api[_-]?key|apikey|secret|password|authtoken|insttoken|access[_-]?token|authorization)$/i;

function safeSourceUrl(value) {
  const url = new URL(value);
  for (const key of [...url.searchParams.keys()]) {
    if (SECRET_QUERY_PARAM_RE.test(key)) url.searchParams.delete(key);
  }
  return String(url);
}

function providerRequest(article, provider, credentials) {
  const doi = encodeURIComponent(article.doi || "");
  if (provider === "elsevier") {
    const headers = { Accept: "application/pdf", "X-ELS-APIKey": credentials.api_key };
    if (credentials.insttoken) headers["X-ELS-Insttoken"] = credentials.insttoken;
    if (credentials.authtoken) headers["X-ELS-Authtoken"] = credentials.authtoken;
    return { url: `https://api.elsevier.com/content/article/doi/${doi}?httpAccept=application/pdf`, options: { headers } };
  }
  if (provider === "springer_nature") {
    const url = new URL("https://spdi.public.springernature.app/xmldata/jats");
    url.searchParams.set("q", `doi:${article.doi}`);
    url.searchParams.set("api_key", credentials.api_key);
    return { url, options: { headers: { Accept: "application/xml" } } };
  }
  if (provider === "ieee") {
    if (!credentials.fulltext_endpoint) {
      return { unavailable: "IEEE metadata API key is configured, but paid Full-Text Access API product/endpoint is missing" };
    }
    const endpoint = credentials.fulltext_endpoint.replace("{doi}", encodeURIComponent(article.doi || ""));
    const url = new URL(endpoint);
    if (!url.searchParams.has("apikey")) url.searchParams.set("apikey", credentials.api_key);
    return { url, options: { headers: { Accept: "application/pdf, text/html, application/xml" } } };
  }
  throw new Error(`unsupported publisher provider: ${provider}`);
}

function failureResult(provider, status, extra = {}) {
  return {
    provider,
    status,
    fallbackConfirmationRequired: status === STATUS.API_NOT_ENTITLED || status === STATUS.API_FULLTEXT_UNAVAILABLE,
    ...extra,
  };
}

export async function downloadPublisherArticle(article, {
  provider,
  credentials,
  outDir,
  fetchImpl = fetch,
  sleepImpl = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
} = {}) {
  if (!credentials?.api_key) {
    return failureResult(provider, STATUS.CREDENTIALS_MISSING, { configureUrl: PROVIDER_CONFIG_URLS[provider] });
  }
  const request = providerRequest(article, provider, credentials);
  if (request.unavailable) return failureResult(provider, STATUS.API_FULLTEXT_UNAVAILABLE, { detail: request.unavailable });
  let response;
  const maxAttempts = provider === "elsevier" ? 3 : 1;
  let lastError = "";
  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
    try {
      response = await fetchImpl(request.url, { ...request.options, signal: AbortSignal.timeout(60000) });
      if (!(provider === "elsevier" && (response.status === 429 || response.status >= 500) && attempt < maxAttempts)) break;
      lastError = `HTTP ${response.status}`;
    } catch (error) {
      lastError = String(error?.message || error).slice(0, 200);
      if (attempt === maxAttempts) break;
    }
    await sleepImpl(250 * attempt);
  }
  if (!response) return failureResult(provider, STATUS.API_FULLTEXT_UNAVAILABLE, { reason: lastError || "request failed", attempts: maxAttempts });
  if (!response.ok) {
    return failureResult(provider, classifyProviderFailure({ status: response.status }), { httpStatus: response.status, attempts: maxAttempts });
  }

  const saved = await saveFullTextResponse(response, {
    outDir,
    title: article.title,
    doi: article.doi,
    source: safeSourceUrl(request.url),
  });
  if (!saved.ok) {
    return failureResult(provider, STATUS.API_FULLTEXT_UNAVAILABLE, {
      httpStatus: saved.httpStatus,
      contentType: saved.contentType,
      reason: saved.reason,
    });
  }
  return {
    provider,
    status: saved.format === "pdf"
      ? STATUS.DOWNLOADED
      : saved.format === "html"
        ? STATUS.FULL_TEXT_HTML_AVAILABLE
        : STATUS.NATIVE_FULLTEXT_DOWNLOADED,
    accessMode: "publisher_api",
    ...saved,
  };
}

import { isCnkiUrl, looksChinese } from "./cnki.mjs";
import { PROVIDER_REGISTRY } from "./provider-registry.mjs";
import { STATUS } from "./status-codes.mjs";

export function parseSiChoice({ si = false, noSi = false } = {}) {
  if (si && noSi) throw new Error("--si and --no-si are mutually exclusive");
  if (!si && !noSi) return { confirmed: false, status: STATUS.SI_CONFIRMATION_REQUIRED };
  return { confirmed: true, wantSi: Boolean(si) };
}

export function classifyPublisher({ doi = "", publisher = "", sourceUrl = "" } = {}) {
  const marker = `${publisher} ${sourceUrl}`.toLowerCase();
  for (const [provider, descriptor] of Object.entries(PROVIDER_REGISTRY)) {
    if (new RegExp(descriptor.doi_pattern, "i").test(doi) || new RegExp(descriptor.marker_pattern, "i").test(marker)) return provider;
  }
  return "other";
}

export function hasUsablePublisherCredentials(provider, credentials) {
  if (!credentials?.api_key) return false;
  return provider !== "ieee" || Boolean(credentials.fulltext_endpoint);
}

export function isChineseLiterature({ title = "", language = "", sourceUrl = "", cnki = null } = {}) {
  if (cnki) return true;
  if (sourceUrl && isCnkiUrl(sourceUrl)) return true;
  if (/^(?:zh|zho|chi)(?:[-_]|$)/i.test(language)) return true;
  return looksChinese(title);
}

export function chooseDirectUrlRoute({ pdfUrl = "", title = "", language = "", sourceUrl = "", route = "" } = {}) {
  if (isCnkiUrl(pdfUrl)) return { provider: "cnki", mode: "direct", reason: "explicit_cnki_url" };
  if (route === "cnki" || isChineseLiterature({ title, language, sourceUrl })) {
    return { provider: "cnki", mode: "title_search", reason: route === "cnki" ? "explicit_override" : "chinese_literature" };
  }
  return { provider: "direct_url", mode: "direct", reason: "explicit_pdf_url" };
}

export function chooseRoute(article = {}) {
  if (article.routeOverride) return { provider: article.routeOverride, reason: "explicit_override" };
  if (isChineseLiterature(article)) return { provider: "cnki", reason: "chinese_literature" };
  const publisher = classifyPublisher(article);
  if (publisher !== "other" && article.hasPublisherCredentials) {
    return { provider: publisher, reason: "publisher_api_credentials_available" };
  }
  if (article.isOa === true) return { provider: "open_access", reason: "article_level_oa" };
  if (publisher !== "other") return { provider: publisher, reason: "supported_publisher" };
  return { provider: "web_access", reason: "other_publisher" };
}

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

export const DEFAULT_DISCOVERY_URL = "https://www.webofscience.com/wos/woscc/basic-search";

export function configPathFromEnv(env = process.env) {
  const dir = env.LIT_DL_CONFIG_DIR || path.join(os.homedir(), ".config", "lit-dl");
  return path.join(dir, "school.json");
}

export function loadSchoolConfig(env = process.env) {
  const file = configPathFromEnv(env);
  if (!fs.existsSync(file)) return null;
  try {
    return JSON.parse(fs.readFileSync(file, "utf8"));
  } catch {
    return null;
  }
}

export function discoveryUrlFromConfig(config) {
  return (
    config?.discovery?.web_of_science_url ||
    config?.discovery?.wos_url ||
    config?.discovery?.url ||
    DEFAULT_DISCOVERY_URL
  );
}

export function schoolSummary(config) {
  if (!config) return "unconfigured";
  const school = config.school?.name || "unknown school";
  const auth = config.auth?.type || "unknown auth";
  return `${school} (${auth})`;
}

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

export function settingsPathFromEnv(env = process.env) {
  const dir = env.LIT_DL_CONFIG_DIR || path.join(os.homedir(), ".config", "lit-dl");
  return path.join(dir, "settings.json");
}

export function loadSettings(env = process.env) {
  const file = settingsPathFromEnv(env);
  if (!fs.existsSync(file)) return {};
  try {
    return JSON.parse(fs.readFileSync(file, "utf8")) || {};
  } catch {
    return {};
  }
}

// Status codes for nature-downloader.
//
// This is the single source of truth for status naming. Both the batch script
// and the manifest/retry TSVs MUST use these codes. The SKILL.md "Status
// Categories" section (L83-98) is the human-readable spec; this file is the
// machine-readable enforcement.
//
// Legacy codes (needs_user_login, needs_user_verify, publisher_blocked,
// no_pdf_link, error) are accepted on input via mapLegacyStatus() for
// backward compatibility with old manifests, but never emitted on output.

export const STATUS = Object.freeze({
  // success
  DOWNLOADED: "downloaded",
  DOWNLOADED_WITH_SI: "downloaded_with_si",
  OPEN_ACCESS_DOWNLOADED: "open_access_downloaded",
  FULL_TEXT_HTML_AVAILABLE: "full_text_html_available",
  AVAILABLE_NOT_DOWNLOADED: "available_not_downloaded",
  NATIVE_FULLTEXT_DOWNLOADED: "native_fulltext_downloaded",

  // user-handoff (not final failure)
  CARSI_WAITING_USER: "carsi_waiting_user",
  CARSI_RESOLVED_RETRY_NEEDED: "carsi_resolved_retry_needed",
  PUBLISHER_VERIFICATION_WAITING_USER: "publisher_verification_waiting_user",
  SCIENCEDIRECT_ROBOT_CHECK: "sciencedirect_robot_check",
  RETRY_AFTER_USER_VERIFICATION: "retry_after_user_verification",
  API_FALLBACK_CONFIRMATION_REQUIRED: "api_fallback_confirmation_required",

  // auto-verification (new — attempted automatic CAPTCHA/slider/robot check solving)
  VERIFICATION_AUTO_PASSED: "verification_auto_passed",
  VERIFICATION_AUTO_FAILED: "verification_auto_failed",

  // do-not-retry
  DO_NOT_AUTO_RETRY: "do_not_auto_retry",
  URL_NEEDS_REPAIR: "url_needs_repair",
  SI_CONFIRMATION_REQUIRED: "si_confirmation_required",
  CREDENTIALS_MISSING: "credentials_missing",
  CREDENTIALS_INVALID: "credentials_invalid",
  API_NOT_ENTITLED: "api_not_entitled",
  API_FULLTEXT_UNAVAILABLE: "api_fulltext_unavailable",
  OA_NOT_FOUND: "oa_not_found",
  OA_RESOLUTION_INCONCLUSIVE: "oa_resolution_inconclusive",
  METADATA_AMBIGUOUS: "metadata_ambiguous",
  RATE_LIMITED: "rate_limited",

  // no access
  LIBRARY_NO_PERMISSION: "library_no_permission",
  NO_FULL_TEXT_LINK: "no_full_text_link",
  PUBLISHER_BLOCKED_WAITING_USER: "publisher_blocked_waiting_user",
  NO_AUTHORIZED_PDF_FOUND: "no_authorized_pdf_found",

  // download failure
  PDF_FETCH_FAILED: "pdf_fetch_failed",
  PDF_CORRUPT: "pdf_corrupt",
  PDF_TOO_SHORT: "pdf_too_short",
  PDF_TOO_LARGE: "pdf_too_large",
  SI_FETCH_FAILED: "si_fetch_failed",
  FAILED_AFTER_RETRY: "failed_after_retry",
});

// Hosts that mean "institutional login wall — stop and hand to user".
const INSTITUTIONAL_HOST_RE =
  /carsi\.edu|\/authserver\/|\/idp\/|\/shibboleth|\/samlsso|\/wayf|\/sso\b/i;

// Publisher anti-bot / verification signals (checked against title + body).
const ROBOT_CHECK_RE =
  /captcha|are you a robot|cloudflare|verify you are human|unusual traffic|bot verification|challenge/i;

// Slider/drag CAPTCHA signals — auto-attemptable, return regular verification status.
const SLIDER_CAPTCHA_RE =
  /滑块验证|滑动验证|拖动滑块|拼图验证|请按住滑块|请拖动|请滑动|drag the slider|slide to verify|slide to unlock|slider verification/i;

// CNKI / Chinese institutional login signals.
const CNKI_LOGIN_RE =
  /登录|统一身份认证|机构登录|校外访问|账号登录|扫码登录|验证码/i;

// Publisher access-denied signals.
const ACCESS_DENIED_RE =
  /access denied|don't have permission|forbidden|403|无权访问|权限不足/i;

/**
 * Classify a wall (login / verification / block) from URL + title + body hint.
 *
 * Returns { status, reason } where status is a canonical STATUS code, or
 * null when no wall is detected.
 *
 * Split logic:
 *  - institutional host in URL  -> carsi_waiting_user
 *  - "Are you a robot?" + sciencedirect host -> sciencedirect_robot_check
 *  - other robot/captcha/cloudflare -> publisher_verification_waiting_user
 *  - access denied / forbidden -> publisher_blocked_waiting_user
 */
export function classifyWall(url, title, bodyHint = "") {
  const u = (url || "").toLowerCase();
  const s = ((title || "") + " " + (bodyHint || "")).toLowerCase();

  // 1. Institutional login wall (CAS / CARSI / Shibboleth / SSO).
  //    Only the URL host decides this — publisher pages legitimately contain
  //    "Log in" links and must not be misclassified as needing the user.
  if (INSTITUTIONAL_HOST_RE.test(u)) {
    return { status: STATUS.CARSI_WAITING_USER, reason: "institutional login wall" };
  }

  // 2. ScienceDirect robot check (specific, worth its own status).
  if (/sciencedirect\.elsevier|sciencedirect\.com/i.test(u) && ROBOT_CHECK_RE.test(s)) {
    return { status: STATUS.SCIENCEDIRECT_ROBOT_CHECK, reason: "ScienceDirect robot check" };
  }

  // 3. Slider / drag CAPTCHA — auto-attemptable, classify as publisher verification.
  if (SLIDER_CAPTCHA_RE.test(s)) {
    return { status: STATUS.PUBLISHER_VERIFICATION_WAITING_USER, reason: "slider captcha — auto-attemptable" };
  }

  // 4. Generic publisher verification (CAPTCHA / Cloudflare / bot check).
  if (ROBOT_CHECK_RE.test(s)) {
    return { status: STATUS.PUBLISHER_VERIFICATION_WAITING_USER, reason: "publisher verification challenge" };
  }

  // 5. Publisher access denied / forbidden / paywall block.
  if (ACCESS_DENIED_RE.test(s)) {
    return { status: STATUS.PUBLISHER_BLOCKED_WAITING_USER, reason: "publisher access denied" };
  }

  return null;
}

// Legacy -> canonical status mapping for backward compatibility with old
// manifests. Used by mapLegacyStatus() when reading old TSV files.
const LEGACY_MAP = Object.freeze({
  needs_user_login: STATUS.CARSI_WAITING_USER,
  needs_user_verify: STATUS.PUBLISHER_VERIFICATION_WAITING_USER, // best-effort; old code didn't distinguish
  publisher_blocked: STATUS.PUBLISHER_BLOCKED_WAITING_USER,
  no_pdf_link: STATUS.NO_FULL_TEXT_LINK, // ambiguous; default to WoS-stage
  error: STATUS.FAILED_AFTER_RETRY,
});

/**
 * Map a legacy status string to its canonical form.
 * Returns the input unchanged if it is already canonical or unknown.
 */
export function mapLegacyStatus(status) {
  if (!status) return status;
  return LEGACY_MAP[status] || status;
}

/**
 * Is this status a "user handoff" (not a final failure)?
 * Used to decide whether to write carsi_retry.tsv / publisher_verification.tsv.
 */
export function isUserHandoff(status) {
  return (
    status === STATUS.CARSI_WAITING_USER ||
    status === STATUS.CARSI_RESOLVED_RETRY_NEEDED ||
    status === STATUS.PUBLISHER_VERIFICATION_WAITING_USER ||
    status === STATUS.SCIENCEDIRECT_ROBOT_CHECK ||
    status === STATUS.RETRY_AFTER_USER_VERIFICATION ||
    status === STATUS.VERIFICATION_AUTO_FAILED
    || status === STATUS.API_FALLBACK_CONFIRMATION_REQUIRED
  );
}

/**
 * Is this status a success?
 */
export function isSuccess(status) {
  return (
    status === STATUS.DOWNLOADED ||
    status === STATUS.DOWNLOADED_WITH_SI ||
    status === STATUS.OPEN_ACCESS_DOWNLOADED ||
    status === STATUS.FULL_TEXT_HTML_AVAILABLE
    || status === STATUS.NATIVE_FULLTEXT_DOWNLOADED
  );
}

import path from "node:path";
import { evalJs, navigate, sleep, waitForComplete } from "./cdp-utils.mjs";
import { fetchAnyToFile } from "./pdf-utils.mjs";

const SUPPLEMENT_LABEL = /(?:supporting\s+information|supplement(?:ary|al)?(?:\s+(?:information|material|data|file|table|figure|video))?|source\s+data|\bmmc\d+\b)/i;
const ATTACHMENT_URL = /(?:downloadSupplement|\/suppl_file\/|_si_|\bmmc\d+\b|\/supplementary\/)/i;
const FILE_EXTENSION = /\.(?:pdf|zip|xlsx?|docx?|csv|tsv|txt|mp4|mov|avi|mpe?g|mp3|wav)(?:$|[?#])/i;
const EXTERNAL_REPOSITORY = /(?:github\.com|zenodo\.org|figshare\.com|dryad\.org|osf\.io)/i;
const CONTENT_TYPE_EXTENSION = new Map([
  ["application/pdf", ".pdf"],
  ["application/zip", ".zip"],
  ["application/x-zip-compressed", ".zip"],
  ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".xlsx"],
  ["application/vnd.ms-excel", ".xls"],
  ["application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx"],
  ["text/csv", ".csv"],
  ["video/mp4", ".mp4"],
]);

function cleanTitle(value = "") {
  return String(value)
    .normalize("NFKC")
    .replace(/[\u2018\u2019]/g, "'")
    .replace(/[^\p{L}\p{N}]+/gu, " ")
    .trim()
    .toLowerCase();
}

export function exactArticleTitleMatch(expected, actual) {
  return Boolean(expected && actual && cleanTitle(expected) === cleanTitle(actual));
}

export function safeArticleTitle(title = "") {
  return String(title)
    .normalize("NFKC")
    .replace(/[\\/:*?"<>|]+/g, "")
    .replace(/\s+/g, " ")
    .trim()
    .slice(0, 160) || "article";
}

export function articleBundleDirectory(outDir, title) {
  return path.join(outDir, safeArticleTitle(title));
}

export function shouldUseCleanWosBundle(args = {}) {
  return Boolean(args.topic && args.title && args.si);
}

export function wosSearchQuery(topic, exactTitle = "") {
  return exactTitle ? `"${String(exactTitle).replace(/"/g, "").trim()}"` : topic;
}

function filenameFromLink(link, url) {
  const explicit = String(link.download || "").trim();
  if (explicit) return path.basename(explicit);
  const parsed = new URL(url);
  const fromQuery = parsed.searchParams.get("file") || parsed.searchParams.get("filename");
  return path.basename(fromQuery || parsed.pathname) || "supplement";
}

export function selectSupportingInformationLinks(links, baseUrl, limit = 30) {
  const attachments = [];
  const pages = [];
  const seen = new Set();
  for (const link of links || []) {
    const rawUrl = link.href || link.dataHref || link.dataUrl || link.dataDownloadUrl || "";
    const label = [link.text, link.ariaLabel, link.title].filter(Boolean).join(" ").trim();
    if (!rawUrl || !SUPPLEMENT_LABEL.test(`${label} ${link.download || ""} ${rawUrl}`)) continue;
    if (/^#/i.test(link.rawHref || "") || /^(?:#|javascript:|mailto:|tel:|data:)/i.test(rawUrl)) continue;
    let url;
    try {
      url = new URL(rawUrl, baseUrl).href;
    } catch {
      continue;
    }
    if (!/^https?:/i.test(url) || EXTERNAL_REPOSITORY.test(url) || seen.has(url)) continue;
    seen.add(url);
    const isAttachment = Boolean(link.download || link.dataDownloadUrl) || FILE_EXTENSION.test(url) || /\/download(?:\/|\?|Supplement)/i.test(url) || ATTACHMENT_URL.test(url) && /(?:file|filename|mmc\d+)/i.test(url);
    if (isAttachment) {
      attachments.push({ url, label, filename: filenameFromLink(link, url) });
    } else {
      pages.push({ url, label });
    }
    if (attachments.length + pages.length >= limit) break;
  }
  return { attachments, pages };
}

const SCAN_LINKS_SCRIPT = `(()=>JSON.stringify(Array.from(document.querySelectorAll('a[href],button,[role=link],[data-href],[data-url],[data-download-url]')).map(e=>({text:(e.innerText||e.textContent||'').trim().slice(0,240),href:e.href||'',rawHref:e.getAttribute('href')||'',download:e.getAttribute('download')||'',ariaLabel:e.getAttribute('aria-label')||'',title:e.getAttribute('title')||'',dataHref:e.getAttribute('data-href')||'',dataUrl:e.getAttribute('data-url')||'',dataDownloadUrl:e.getAttribute('data-download-url')||''}))))()`;

async function scanPage(proxy, tab, url) {
  await navigate(proxy, tab, url);
  await waitForComplete(proxy, tab);
  let links = [];
  for (let attempt = 0; attempt < 5; attempt++) {
    await sleep(attempt === 0 ? 800 : 1000);
    const raw = await evalJs(proxy, tab, SCAN_LINKS_SCRIPT);
    try {
      links = JSON.parse(raw || "[]");
    } catch {
      links = [];
    }
    const found = selectSupportingInformationLinks(links, url);
    if (found.attachments.length || found.pages.length) break;
  }
  return links;
}

export async function fetchAttachmentWithNavigationFallback(
  proxy,
  tab,
  attachment,
  resolvePath,
  dependencies = {}
) {
  const fetchImpl = dependencies.fetchImpl || ((p, t, url, outPath) => (
    fetchAnyToFile(p, t, url, outPath, { rejectHtml: true })
  ));
  const navigateImpl = dependencies.navigateImpl || navigate;
  const waitForCompleteImpl = dependencies.waitForCompleteImpl || waitForComplete;
  let result = await fetchImpl(proxy, tab, attachment.url, resolvePath);
  if (result.ok || !/(?:failed to fetch|cors)/i.test(result.err || "")) return result;
  await navigateImpl(proxy, tab, attachment.url);
  await waitForCompleteImpl(proxy, tab);
  return fetchImpl(proxy, tab, attachment.url, resolvePath);
}

function filenameFromDisposition(value = "") {
  const utf8 = value.match(/filename\*\s*=\s*UTF-8''([^;]+)/i);
  if (utf8) {
    try {
      return decodeURIComponent(utf8[1].replace(/^"|"$/g, ""));
    } catch {}
  }
  return (value.match(/filename\s*=\s*"([^"]+)"/i) || value.match(/filename\s*=\s*([^;]+)/i) || [])[1] || "";
}

function safeAttachmentName(value = "") {
  return path.basename(String(value))
    .replace(/[\\/:*?"<>|]+/g, "_")
    .replace(/^\.+/, "")
    .trim()
    .slice(0, 140) || "supplement";
}

function uniqueAttachmentPath(bundleDir, attachment, meta, usedNames) {
  const headerName = filenameFromDisposition(meta.contentDisposition);
  let filename = safeAttachmentName(headerName || attachment.filename);
  if (!path.extname(filename)) {
    let finalName = "";
    try {
      finalName = path.basename(new URL(meta.finalUrl || meta.url || attachment.url).pathname);
    } catch {}
    if (path.extname(finalName)) filename = safeAttachmentName(finalName);
    else {
      const contentType = String(meta.contentType || "").split(";")[0].trim().toLowerCase();
      filename += CONTENT_TYPE_EXTENSION.get(contentType) || "";
    }
  }
  let candidate = filename;
  let suffix = 2;
  while (usedNames.has(candidate.toLowerCase())) {
    const ext = path.extname(filename);
    const stem = filename.slice(0, filename.length - ext.length);
    candidate = `${stem}-${suffix++}${ext}`;
  }
  usedNames.add(candidate.toLowerCase());
  return path.join(bundleDir, candidate);
}

export async function downloadWosSupportingInformation({
  proxy,
  tab,
  landingUrl,
  bundleDir,
  reservedFilenames = [],
  dependencies = {},
}) {
  const scanPageImpl = dependencies.scanPageImpl || scanPage;
  const fetchAttachmentImpl = dependencies.fetchAttachmentImpl || fetchAttachmentWithNavigationFallback;

  let landingLinks;
  try {
    landingLinks = await scanPageImpl(proxy, tab, landingUrl);
  } catch (error) {
    return { status: "fetch_failed", found: 0, downloaded: 0, files: [], failures: [{ url: landingUrl, error: String(error).slice(0, 200) }] };
  }
  const first = selectSupportingInformationLinks(landingLinks, landingUrl);
  const attachments = [...first.attachments];
  const seen = new Set(attachments.map((item) => item.url));
  const failures = [];

  for (const page of first.pages) {
    try {
      const nestedLinks = await scanPageImpl(proxy, tab, page.url);
      const nested = selectSupportingInformationLinks(nestedLinks, page.url);
      for (const attachment of nested.attachments) {
        if (seen.has(attachment.url)) continue;
        seen.add(attachment.url);
        attachments.push(attachment);
        if (attachments.length >= 30) break;
      }
    } catch (error) {
      failures.push({ url: page.url, error: String(error).slice(0, 200) });
    }
    if (attachments.length >= 30) break;
  }

  if (!attachments.length) {
    return {
      status: failures.length ? "fetch_failed" : "not_found",
      found: 0,
      downloaded: 0,
      files: [],
      failures,
    };
  }

  const files = [];
  const usedNames = new Set(reservedFilenames.map((name) => String(name).toLowerCase()));
  for (const attachment of attachments) {
    const result = await fetchAttachmentImpl(
      proxy,
      tab,
      attachment,
      (meta) => uniqueAttachmentPath(bundleDir, attachment, meta, usedNames)
    ).catch((error) => ({ ok: false, err: String(error).slice(0, 200) }));
    if (result.ok) files.push(result.file);
    else failures.push({ url: attachment.url, error: result.err || "download failed" });
  }
  const downloaded = files.length;
  const status = downloaded === attachments.length && failures.length === 0 ? "downloaded" : downloaded > 0 ? "partial" : "fetch_failed";
  return { status, found: attachments.length, downloaded, files, failures };
}

#!/usr/bin/env node
// Deprecated compatibility wrapper. The canonical Elsevier implementation is
// scripts/lib/publisher-providers.mjs, orchestrated by batch_download.mjs.

import { spawnSync } from "node:child_process";
import path from "node:path";
import { fileURLToPath } from "node:url";

const here = path.dirname(fileURLToPath(import.meta.url));
const args = process.argv.slice(2);
if (!args.length) {
  console.error("retry_elsevier_api.mjs is deprecated. Pass normal batch_download arguments, including --dois, --out, and exactly one of --si/--no-si.");
  process.exit(2);
}
if (!args.includes("--route")) args.push("--route", "elsevier");
const result = spawnSync(process.execPath, [path.join(here, "batch_download.mjs"), ...args], { stdio: "inherit" });
process.exit(result.status ?? 1);

---
name: nature-downloader
description: Use when a user needs lawful academic full text, CNKI institutional access, English OA retrieval, publisher API access, institutional browser fallback, or supporting information downloads.
metadata:
  compatibility: Requires Node.js 22+ and Python 3. CNKI, Web Access, and SI routes additionally require the user's authenticated Chrome session and remote debugging. Uses only lawful OA, publisher API, and user-authorized institutional access.
---

# Nature Literature Downloader

This skill routes literature through lawful open-access, publisher-API, CNKI institutional, and browser-based institutional providers. `scripts/batch_download.mjs` is the orchestration entry point; school configuration, publisher credentials, metadata/OA resolution, provider downloads, content validation, and manifests are separate modules.

Verified routes are examples, not defaults. Every institution should start from the user's actual library resource URL, because resource portals, CAS callbacks, EZproxy, WebVPN, IP-authenticated database pages, and database detail pages reveal the live authorization path more reliably than a school name.

> **SI confirmation gate — do this first.** Before downloading any PDF, CAJ, HTML, XML, archive, or attachment, ask whether the user wants Supporting Information. An explicit request for SI counts as yes; an explicit request for正文 only counts as no. Otherwise ask once for the whole batch. Run the downloader with exactly one of `--si` or `--no-si`. Without either flag the script returns `si_confirmation_required` and does not create the output directory.

> **Main workflow.** Normalize the DOI/title and identify language and publisher before routing. Chinese literature always uses CNKI. For English Elsevier, Springer Nature, and IEEE articles with usable provider credentials, try the publisher API first and do not require an OA determination after a successful API download. If that API attempt fails, automatically check legitimate OA sources. Other English publishers check OA first, then use the institutional Web Access route when OA is unavailable.

```text
规范化 DOI/题名并识别语言、出版商
├─ 中文文献：直接走 CNKI
└─ 英文文献
   ├─ Elsevier / Springer Nature / IEEE，且已配置有效 Key
   │  ├─ 优先通过出版商 API 下载
   │  ├─ API 下载成功：结束，不强制判断 OA
   │  └─ API 下载失败：检查文章级 OA，再走 PMC / Unpaywall / 合法仓储
   └─ 其他出版商
      ├─ 检查文章级 OA
      └─ OA 不可用：走 Web Access 机构授权
```

> **Chinese literature is CNKI-only.** A Chinese title, `zh` metadata language, explicit CNKI source URL, or `--route cnki` must use CNKI even if another OA copy appears to exist. Reuse the user's current Chrome library/CNKI login state and prefer configured `discovery.cnki_url`. Never export cookies or collect the institutional password.

> **Publisher API fallback.** A valid API key does not guarantee full-text entitlement. When an Elsevier, Springer Nature, or IEEE API attempt returns no entitlement or no usable full text, automatically try legitimate OA sources first. Return `api_fallback_confirmation_required` and ask once whether to use Web Access only after both the publisher API and OA routes fail. Do not switch to institutional Web Access automatically.

> **Browser-state principle.** Authorized downloads depend on the exact browser profile where the user is logged in. If a proxy, CDP session, or browser automation tool opens a fresh profile or a different browser with no login state, do not treat the failure as missing library permission. Switch to a control path that reuses the user's active browser session, or ask the user to authenticate in the controlled browser instance.

> **Format principle.** PDF, HTML full text, and database-native formats such as CAJ are different deliverables. If the user asks for PDF only, require a real PDF link or `%PDF` response and report `no_authorized_pdf_found` / `pdf_fetch_failed` when none exists. Do not save CAJ, HTML, or a login page as if it were a PDF.

## Download Intake and First-Run Configuration

For every download request, first establish the paper list and ask:

```text
是否同时下载这些文献的 Supporting Information（SI，补充材料）？
```

Do metadata lookup before this question only when needed to identify the requested papers. Do not download files until the answer is known. Configure a library only when the selected route is CNKI or Web Access. Configure a publisher API only when the selected English article belongs to Elsevier, Springer Nature, or IEEE; an OA determination is not required before trying a configured provider API.

### Paid Library Resource Configuration

Ask for the library resource URL the user actually uses:

```text
请发你平时进入图书馆电子资源/数据库的平台链接。
可以是资源门户、数据库列表、Web of Science 入口、某个数据库详情页，
或跳转到统一身份认证的登录链接。
```

Then infer the authorization route from the URL before saving config:

```bash
python3 scripts/configure_school.py infer "https://example.edu/library/resources"
python3 scripts/configure_school.py url "https://example.edu/library/resources"
python3 scripts/configure_school.py show
python3 scripts/configure_school.py health --force
```

The distributed skill contains no school presets. If the user cannot provide a resource URL, ask them to locate their institution's library/database entry instead of guessing a school-specific domain.

The default config path is:

```text
~/.config/lit-dl/school.json
```

For tests or isolated profiles, set:

```bash
LIT_DL_CONFIG_DIR=/path/to/configdir
```

The downloader reads this config automatically. If `discovery.web_of_science_url` is present, `scripts/batch_download.mjs` uses it as the Web of Science entry; otherwise it falls back to `https://www.webofscience.com/wos/woscc/basic-search`.

For Chinese literature, the downloader also reads `discovery.cnki_url` when present. If absent, `scripts/batch_download.mjs --title "<中文题名>"` falls back to `https://kns.cnki.net/kns8s/defaultresult/index`.

### API-First and Open-Access Fallback

For an English article, identify its publisher before deciding when to resolve article-level OA:

1. Collect a DOI, PMID, exact title, article URL, or a definite paper list, then normalize its metadata and publisher.
2. If it belongs to Elsevier, Springer Nature, or IEEE and usable provider credentials are configured, try that publisher API first. On success, record `accessMode: publisher_api` and `oa_status: not_checked_api_first`; do not run OA resolution only to label the article.
3. If the publisher API fails, automatically search legitimate OA sources such as PMC, Unpaywall, publisher OA pages, arXiv, and other lawful repositories or clearly open PDF URLs. Preserve the failed API attempt in the manifest.
4. For all other English publishers, search those legitimate OA sources before Web Access.
5. For an exact title or an explicit OA-only request, prefer:

   ```bash
   node scripts/batch_download.mjs --title "<exact title>" --open-access --no-si --out "<project>"
   ```

   Use `--pdf-url` when the user supplies a known legitimate OA PDF URL.
6. Verify the downloaded file and record the source. Mark a successful PDF as `open_access_downloaded`.
7. If no lawful OA full text is found, mark `oa_not_found`. For a supported publisher whose API already failed, request confirmation before Web Access. For another publisher, continue to Web Access. If `--route open_access` was explicitly requested, stop after the OA result.

### Publisher API Credentials

Configure credentials lazily, only when the route first needs them:

```bash
python3 scripts/configure_credentials.py set elsevier
python3 scripts/configure_credentials.py set springer_nature
python3 scripts/configure_credentials.py set ieee --fulltext-endpoint 'https://issued-endpoint.example/articles/{doi}'
python3 scripts/configure_credentials.py set elsevier --stdin
python3 scripts/configure_credentials.py show
python3 scripts/configure_credentials.py validate <provider>
python3 scripts/configure_credentials.py delete <provider>
python3 scripts/configure_credentials.py contact-email researcher@example.org
```

Give the user the official registration link: Elsevier `https://dev.elsevier.com/`, Springer Nature `https://dev.springernature.com/docs/quick-start/api-access/`, or IEEE `https://developer.ieee.org/member/register`.

Do not proactively ask the user to paste an API key into chat. If the user voluntarily sends a publisher API key, treat that as authorization to save that exact key: do not reject it, ask them to regenerate it, or repeat it back. Pass it to `configure_credentials.py set <provider> --stdin`, keep it out of command-line arguments, logs, replies, and manifests, then report only the masked confirmation and validation status. The local hidden prompt remains the preferred path when the key has not already been provided. IEEE Metadata API access is not paid full-text access; require the issued Full-Text Access endpoint/template before treating IEEE as downloadable through the API. Secrets are stored in `~/.config/lit-dl/credentials.json` with mode `0600`.

## Resource URL Triage

Classify the user-provided URL before choosing an access path:

```text
cas.* / /authserver/login        CAS / SSO login page; inspect service= callback, then return to the service portal
idp/shibboleth / carsi           CARSI / Shibboleth institutional route
ezproxy / libproxy               EZproxy remote-access proxy
webvpn / vpn                     WebVPN route
metaersp / metaauth / uas        Library resource aggregation portal
webofscience / sciencedirect     Database or publisher entry; check whether it was reached through a portal
```

If the URL is a login page with a `service=` parameter, treat the callback host as the resource service and do not make the login page the whole workflow. For example, `https://login.university.example/authserver/login?service=https://resources.university.example/callback` means the identity service returns to the user's resource portal after authentication.

## Institution-Specific Domains

Confirm against what actually appears in the user's address bar; correct these for each institution instead of assuming a preset is complete.

```text
Library home / aggregation:  library.example.edu, resources.example.edu
Discovery/database entry:    webofscience.com, clarivate.com, cnki.net, sciencedirect.com, provider.example.com
Unified identity / SSO:      sso.example.edu, cas.example.edu, idp.example.edu
Federation / WAYF:           ds.carsi.edu.cn, wayf.example.org, shibboleth/openathens hosts
Proxy / WebVPN:              ezproxy.example.edu, webvpn.example.edu
```

Treat configured institutional login, federation, proxy, and database-login hosts as sign-in stages. Do not treat reaching them as a final failure.

## Boundaries

Use only the user's legitimate institutional access. Do not bypass paywalls, DRM, or two-factor authentication.

**Verification-first rule:** When a visible slider, checkbox, robot check, or simple verification control appears in the user's authenticated Chrome session, attempt it in the browser before asking the user to intervene. Keep the attempt bounded (at most two attempts on one tab), verify that the challenge disappeared, and continue from that same tab when successful.

- Slider/drag challenges (including CNKI puzzle sliders): estimate the visible travel distance and simulate a gradual drag.
- ScienceDirect robot checks, managed Turnstile, and reCAPTCHA checkbox stages: try the visible checkbox once.
- Simple `Continue`, `Verify`, or equivalent visible controls: click once, then re-check the page state.

**User handoff:** Ask the user only after the bounded attempt fails, or immediately when the page requires secret or identity-bearing input such as an image-selection answer, QR approval, SMS/OTP, passkey, hardware key, or two-factor authentication. Keep the challenged tab open and never ask the user to paste credentials or codes into chat.

Avoid unbounded or indiscriminate downloading. Process only the definite paper list confirmed by the user, apply provider-friendly pacing, and leave a clear audit trail of what was downloaded, from where, and whether supporting information was found.

Do not ask the user to paste institutional passwords, database passwords, OTP codes, recovery codes, or session tokens into chat or terminal. If the user offers one of those identity-bearing secrets, decline and use the handoff-login workflow instead. Publisher API keys follow the separate save-on-receipt rule above.

Exception for saved institutional login pages: if the user explicitly says that the browser has already filled credentials and authorizes clicking the visible login/confirm button, the agent may click that button once on the expected institutional SSO / CAS / CARSI / Shibboleth page without reading, copying, or typing any credential. This exception does not apply to CAPTCHA, QR login, SMS/OTP, publisher bot checks, consent/security warnings, or any page outside the expected institutional login flow.

Do not inspect or export cookies, passwords, local storage, browser profiles, or session files. Use the browser's already-authenticated page context only.

## Preconditions

Before attempting downloads, confirm the conditions that apply to the selected access branch.

For the OA-only branch, confirm the target paper identifier/list, output folder, Node.js 22+, and Python 3 when PDF verification needs it. Do not require a library configuration or institutional browser login.

For the paid-library branch, confirm these conditions:

1. The browser that holds the user's library/database login state is open on the user's machine.
2. The school configuration exists and is valid.
   - Run `python3 scripts/configure_school.py show`.
   - If missing, run `python3 scripts/configure_school.py preset "<school name>"` or guide the user through `src/wizard.py`.
3. The user has personally logged in to their institution/library route in that same browser, and can reach the library aggregation service, target database, or discovery entry.
4. The browser-control path can reuse that same logged-in browser profile.
   - For Chrome CDP, ask the user to open `chrome://inspect/#remote-debugging` and enable remote debugging for the current browser instance.
   - If CDP attaches to a stale browser, a temporary profile, or a different browser, use a browser-control channel that can reuse the user's active session instead of launching a new profile.
5. The environment can run Node.js 22+.
   - Try `node --version`.
   - If `node` is not on PATH in Codex Desktop, try `%LOCALAPPDATA%\OpenAI\Codex\bin\node.exe`.
6. The environment can run Python 3 for configuration and PDF text verification.
   - Try `python3 --version`.
   - Install Python helpers with `pip install -r requirements.txt` when needed.
7. The web-access CDP proxy is available or can be started.
   - Typical Claude Code path: `%USERPROFILE%\.claude\skills\web-access-main\scripts\check-deps.mjs`.
   - Typical shared agent path: `%USERPROFILE%\.agents\skills\web-access-main\scripts\check-deps.mjs`.
   - In Codex-only setups also check `%USERPROFILE%\.codex\skills\web-access-main\scripts\check-deps.mjs`.
8. The user has approved the target output folder.

If Claude Code says this skill is not installed, install or copy it to:

```powershell
$env:USERPROFILE\.claude\skills\nature-downloader
```

Codex and other agent setups may instead use `.codex\skills` or `.agents\skills`; treat the three locations as install targets, not as different skill versions.

## Batch Scope

Definite DOI/title/PMID lists are supported without a fixed per-batch paper-count recommendation.

Operational safeguards:

- pace requests appropriately for each provider and maintain the manifest throughout the batch
- attempt visible verification controls first; stop after at most two failed attempts, on institutional login expiry, or when an unusual/security-sensitive prompt appears

Do not turn a broad keyword search into unlimited automatic downloading. Do not download whole journal issues, volumes, or large result sets.

## Status Categories

Classify every paper into one of these statuses, and keep the status in the manifest:

```text
downloaded
downloaded_with_si
open_access_downloaded
full_text_html_available
available_not_downloaded
native_fulltext_downloaded
si_confirmation_required
credentials_missing
credentials_invalid
api_not_entitled
api_fulltext_unavailable
api_fallback_confirmation_required
oa_not_found
oa_resolution_inconclusive
metadata_ambiguous
carsi_waiting_user
carsi_resolved_retry_needed
publisher_verification_waiting_user
sciencedirect_robot_check
retry_after_user_verification
verification_auto_passed
verification_auto_failed
do_not_auto_retry
url_needs_repair
library_no_permission
no_full_text_link
publisher_blocked_waiting_user
no_authorized_pdf_found
failed_after_retry
```

Use `verification_auto_passed` when an automatic CAPTCHA/slider/robot check was successfully solved by the skill, and the download then proceeded normally.

Use `verification_auto_failed` when auto-verification was attempted but could not pass the challenge. This is a user-handoff status, not a final failure.

Use `carsi_waiting_user` only when the browser is visibly at an institutional SSO / CAS / CARSI-Shibboleth / OpenAthens / database authentication page. Do not treat this as a final failure.

Use `publisher_verification_waiting_user` or `sciencedirect_robot_check` when a publisher page shows a verification challenge but no automatic interaction was possible. When a bounded automatic attempt was made and failed, use `verification_auto_failed` instead. None of these is a final download failure.

Use `open_access_downloaded` when a legitimate open-access route such as PMC, the publisher's OA PDF, arXiv, or another lawful open PDF source provides the downloaded PDF without institutional authorization.

For a successful API-first download, record `oa_status: not_checked_api_first`; this means OA resolution was intentionally skipped, not that the article is non-OA. Use `api_fallback_confirmation_required` only after a supported publisher API attempt and its automatic OA fallback both fail.

Use `full_text_html_available` when the library/full-text resolver grants access to a readable HTML full text but no valid PDF link or `%PDF` response is available. This is a successful full-text access result, not a PDF download. Save the HTML/text if the user asked for the article, and explicitly tell the user that the PDF was not available through the current authorized route.

Use `library_no_permission` when the library portal, SFX/OpenURL resolver, database, or publisher page clearly says the user's institution has no full-text entitlement for the paper. Tell the user plainly that the current library resources do not have permission for this article. Do not retry direct publisher access as if it were a temporary network problem.

## Start Browser Control

Use the web-access CDP proxy when it can attach to the same logged-in browser instance the user is using. If the task depends on existing login state and CDP opens a blank/new profile, prefer a browser-control channel that reuses the user's active browser session.

On Windows PowerShell:

```powershell
$node = "node"
if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
  $node = "$env:LOCALAPPDATA\OpenAI\Codex\bin\node.exe"
}
$checkDepsCandidates = @(
  "$env:USERPROFILE\.claude\skills\web-access-main\scripts\check-deps.mjs",
  "$env:USERPROFILE\.agents\skills\web-access-main\scripts\check-deps.mjs",
  "$env:USERPROFILE\.codex\skills\web-access-main\scripts\check-deps.mjs"
)
$checkDeps = $checkDepsCandidates | Where-Object { Test-Path $_ } | Select-Object -First 1
if (-not $checkDeps) { throw "web-access-main/scripts/check-deps.mjs not found" }
& $node $checkDeps
```

Then test:

```powershell
Invoke-WebRequest -UseBasicParsing -Uri "http://127.0.0.1:3456/targets" -TimeoutSec 10
```

If this hangs or fails:

- Ask the user to confirm the remote debugging checkbox.
- Check `%TEMP%\cdp-proxy.log`.
- If targets appear but the database/library page is unauthenticated, suspect a stale CDP endpoint, wrong browser, or fresh browser profile before suspecting missing library permission.
- Do not attempt to read Chrome session files.

## Fast Batch Path (default for 2+ papers — fast & token-efficient)

For anything beyond a single paper, run `scripts/batch_download.mjs` instead of driving the browser step-by-step. OA and publisher APIs run without CDP; CNKI, Web Access, and requested SI lazily attach to the authenticated browser. Large DOMs and file bytes remain inside the scripts.

The script reads `~/.config/lit-dl/school.json` automatically. When the config contains `discovery.web_of_science_url`, that URL is used as the Web of Science entry; otherwise the script falls back to its compiled default Web of Science URL.

```bash
# by topic (collects N records from Web of Science Core Collection):
node scripts/batch_download.mjs --topic "rice blast resistance gene" --count 10 --no-si --out "<project>"
# by explicit DOIs:
node scripts/batch_download.mjs --dois "10.1007/s00122-021-03957-1,10.1111/pbi.14066" --no-si --out "<project>"
# by exact open-access title (arXiv fallback, useful for DOI-less papers):
node scripts/batch_download.mjs --title "Attention Is All You Need" --open-access --no-si --out "<project>"
# by Chinese exact title (default CNKI route):
node scripts/batch_download.mjs --title "乡村振兴背景下数字治理研究" --no-si --out "<project>"
# by Chinese exact title, PDF only:
node scripts/batch_download.mjs --title "乡村振兴背景下数字治理研究" --cnki-format pdf --no-si --out "<project>"
# by Chinese exact title with a library-provided CNKI entry:
node scripts/batch_download.mjs --title "乡村振兴背景下数字治理研究" --cnki-url "https://kns.cnki.net/kns8s/defaultresult/index" --no-si --out "<project>"
# by known PDF URL:
node scripts/batch_download.mjs --pdf-url "https://arxiv.org/pdf/1706.03762" --title "Attention Is All You Need" --no-si --out "<project>"
# replace --no-si with --si only after the user explicitly requests SI
```

Output includes `{ summary, manifest, results }`. The script writes `<project>/manifest.json` with route, OA evidence, access mode, format, MIME, bytes, SHA-256, SI choice, and typed failures; secret-looking fields are removed recursively. PDFs go under `PDFs/`, native HTML/XML under `FullText/`, CAJ under `CNKI/`, and supplements under `SupportingInformation/`.

**Token discipline (applies to all paths):** never `eval` a whole page DOM, search result, or PDF/SI bytes back into the agent context. Keep large data inside Node/`scripts/*.mjs` and surface only compact status. Reserve interactive `/eval` + `cdp_open_url.mjs` for the single-paper route below or for diagnosing one stuck paper after the batch run.

## Advanced Browser and Delivery Routes

Keep this router compact and load the detailed operational references only when their conditions
apply:

- Load [references/institutional-browser-workflow.md](references/institutional-browser-workflow.md)
  when legitimate OA and applicable publisher-API routes are exhausted and the task needs Web of
  Science, an institution-authorized browser session, publisher verification, an authentication
  handoff, or browser-context PDF transfer.
- Load
  [references/delivery-verification-and-failures.md](references/delivery-verification-and-failures.md)
  when the user requests Supporting Information, downloaded files need final verification and
  naming, or an access attempt reaches a typed failure or retry state.

The boundaries, SI confirmation gate, browser-state principle, status semantics, and token
discipline in this router remain mandatory when either reference is loaded. Do not treat the
references as permission to bypass access controls or to expose credentials, cookies, or session
data.

"""配置读写模块。

负责 school.json 的读取、写入、校验、备份。
配置路径：~/.config/lit-dl/school.json
"""

from __future__ import annotations

import json
import os
import shutil
from datetime import datetime
from pathlib import Path
from typing import Any, Optional

try:
    import jsonschema  # type: ignore
except ImportError:
    jsonschema = None  # type: ignore

# 配置文件路径（支持环境变量覆盖，便于测试和多 profile）
CONFIG_DIR = Path(os.environ.get("LIT_DL_CONFIG_DIR", Path.home() / ".config" / "lit-dl"))
CONFIG_FILE = CONFIG_DIR / "school.json"

# schema 路径（相对于本文件）
SCHEMA_FILE = Path(__file__).resolve().parent.parent / "data" / "school.schema.json"


def load_schema() -> dict[str, Any]:
    """加载 JSON Schema。"""
    with open(SCHEMA_FILE, "r", encoding="utf-8") as f:
        return json.load(f)


def config_exists() -> bool:
    """配置文件是否存在。"""
    return CONFIG_FILE.exists()


def validate(config: dict[str, Any]) -> list[str]:
    """校验配置是否符合 schema。

    返回错误消息列表，空列表表示通过。
    若 jsonschema 未安装则跳过 schema 校验，只做基本字段检查。
    """
    errors: list[str] = []

    # 基本字段检查（不依赖 jsonschema）
    if not isinstance(config, dict):
        return ["配置不是合法的 JSON 对象"]
    if "version" not in config:
        errors.append("缺少 version 字段")
    if "school" not in config or "name" not in config.get("school", {}):
        errors.append("缺少 school.name 字段")
    auth = config.get("auth", {})
    if "type" not in auth:
        errors.append("缺少 auth.type 字段")
    if "sso_domain" not in auth or not auth["sso_domain"]:
        errors.append("缺少 auth.sso_domain 字段")
    if not config.get("libraries"):
        errors.append("libraries 不能为空")

    # schema 校验（可选）
    if jsonschema is not None and not errors:
        try:
            jsonschema.validate(instance=config, schema=load_schema())
        except jsonschema.ValidationError as e:  # type: ignore
            errors.append(f"schema 校验失败：{e.message}")

    return errors


def load_config() -> Optional[dict[str, Any]]:
    """读取配置文件。

    返回配置字典，文件不存在返回 None。
    若 JSON 解析失败，自动备份旧文件并返回 None。
    """
    if not CONFIG_FILE.exists():
        return None

    try:
        with open(CONFIG_FILE, "r", encoding="utf-8") as f:
            return json.load(f)
    except (json.JSONDecodeError, UnicodeDecodeError) as e:
        # 配置文件损坏，备份后返回 None
        backup = CONFIG_FILE.with_suffix(".json.broken")
        shutil.copy2(CONFIG_FILE, backup)
        try:
            CONFIG_FILE.unlink()
        except OSError:
            pass
        print(f"配置文件损坏，已备份到 {backup}，请重新配置。错误：{e}")
        return None


def save_config(config: dict[str, Any]) -> Path:
    """写入配置文件。

    自动创建目录，补充 configured_at 时间戳。
    返回配置文件路径。
    """
    CONFIG_DIR.mkdir(parents=True, exist_ok=True)

    # 补充时间戳
    if not config.get("school", {}).get("configured_at"):
        config.setdefault("school", {})["configured_at"] = datetime.now().isoformat()

    # 校验
    errors = validate(config)
    if errors:
        raise ValueError(f"配置校验失败：{'; '.join(errors)}")

    with open(CONFIG_FILE, "w", encoding="utf-8") as f:
        json.dump(config, f, ensure_ascii=False, indent=2)

    # 设置文件权限（仅属主可读写）
    os.chmod(CONFIG_FILE, 0o600)

    return CONFIG_FILE


def backup_config() -> Optional[Path]:
    """备份当前配置文件，返回备份路径。文件不存在返回 None。"""
    if not CONFIG_FILE.exists():
        return None
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    backup = CONFIG_FILE.with_suffix(f".json.{timestamp}.bak")
    shutil.copy2(CONFIG_FILE, backup)
    return backup


def delete_config() -> bool:
    """删除配置文件（用于重新配置）。返回是否删除成功。"""
    if CONFIG_FILE.exists():
        backup_config()
        CONFIG_FILE.unlink()
        return True
    return False


def get_school_name() -> Optional[str]:
    """快捷获取学校名称。未配置返回 None。"""
    cfg = load_config()
    if cfg is None:
        return None
    return cfg.get("school", {}).get("name")


def get_auth_info() -> Optional[dict[str, Any]]:
    """快捷获取认证信息。未配置返回 None。"""
    cfg = load_config()
    if cfg is None:
        return None
    return cfg.get("auth")


if __name__ == "__main__":
    # CLI 自检
    if config_exists():
        cfg = load_config()
        if cfg:
            print(f"已配置学校：{cfg.get('school', {}).get('name', '未知')}")
            print(f"配置路径：{CONFIG_FILE}")
            errors = validate(cfg)
            if errors:
                print(f"校验警告：{errors}")
            else:
                print("配置校验通过")
        else:
            print("配置文件存在但损坏，请重新配置")
    else:
        print(f"尚未配置，配置文件路径：{CONFIG_FILE}")

"""连通性自检模块。

下载前对配置做轻量可达性探测，结果缓存 10 分钟。
失败时给出具体排查建议。
"""

from __future__ import annotations

import json
import time
from pathlib import Path
from typing import Any, Optional

from config import CONFIG_DIR, load_config
from validators import validate_carsi_entry, validate_sso_domain

# 缓存文件
CACHE_FILE = CONFIG_DIR / "health_cache.json"
CACHE_TTL = 600  # 10 分钟


def _load_cache() -> Optional[dict[str, Any]]:
    if not CACHE_FILE.exists():
        return None
    try:
        with open(CACHE_FILE, "r", encoding="utf-8") as f:
            return json.load(f)
    except (json.JSONDecodeError, OSError):
        return None


def _save_cache(data: dict[str, Any]) -> None:
    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
    with open(CACHE_FILE, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)


def _clear_cache() -> None:
    if CACHE_FILE.exists():
        CACHE_FILE.unlink()


def _diagnose_failure(cfg: dict[str, Any]) -> list[str]:
    """根据配置内容给出排查建议。"""
    suggestions: list[str] = []
    auth = cfg.get("auth", {})
    sso_domain = auth.get("sso_domain", "")
    carsi_entry = auth.get("carsi_entry", "")

    suggestions.append("可能原因与建议：")

    if sso_domain:
        suggestions.append(
            f"1. 检查网络：当前是否在校园网内或已连 VPN？"
            f"校外访问 {sso_domain} 可能需要 VPN。"
        )
    if carsi_entry:
        suggestions.append(
            f"2. CARSI 入口可能变更：访问 https://www.carsi.edu.cn/ "
            f"确认贵校当前入口，或说「换学校」重新配置。"
        )
    suggestions.append("3. 学校认证服务可能临时不可用，稍后重试。")
    suggestions.append("4. 如持续失败，说「重新配置」进入向导修正参数。")

    return suggestions


def health_check(force: bool = False) -> dict[str, Any]:
    """执行连通性自检。

    参数：
        force: 是否跳过缓存强制检测

    返回：
        {
            "ok": bool,
            "checked_at": str,
            "cached": bool,
            "details": [...],
            "suggestions": [...]  # 仅失败时
        }
    """
    # 检查缓存
    if not force:
        cache = _load_cache()
        if cache and (time.time() - cache.get("checked_at_ts", 0)) < CACHE_TTL:
            cache["cached"] = True
            return cache

    cfg = load_config()
    if cfg is None:
        return {
            "ok": False,
            "checked_at": time.strftime("%Y-%m-%d %H:%M:%S"),
            "cached": False,
            "details": ["未找到配置，请先运行配置向导"],
            "suggestions": ["说「配置学校」或「/reconfig」进入配置向导"],
        }

    details: list[str] = []
    all_ok = True

    auth = cfg.get("auth", {})
    sso_domain = auth.get("sso_domain", "")
    carsi_entry = auth.get("carsi_entry", "")

    # 1. SSO 域名探测
    if sso_domain:
        ok, msg = validate_sso_domain(sso_domain)
        details.append(f"[SSO] {msg}")
        if not ok:
            all_ok = False
    else:
        details.append("[SSO] 未配置 sso_domain")
        all_ok = False

    # 2. CARSI 入口探测（如配置了）
    if carsi_entry:
        ok, msg = validate_carsi_entry(carsi_entry)
        details.append(f"[CARSI] {msg}")
        if not ok:
            all_ok = False
    else:
        details.append("[CARSI] 未配置 CARSI 入口（如该校无 CARSI 可忽略）")

    result: dict[str, Any] = {
        "ok": all_ok,
        "checked_at": time.strftime("%Y-%m-%d %H:%M:%S"),
        "checked_at_ts": time.time(),
        "cached": False,
        "details": details,
    }

    if not all_ok:
        result["suggestions"] = _diagnose_failure(cfg)

    # 写缓存
    _save_cache(result)

    return result


def clear_cache() -> None:
    """清除自检缓存（配置变更后调用）。"""
    _clear_cache()


if __name__ == "__main__":
    result = health_check(force=True)
    print(f"自检结果：{'通过' if result['ok'] else '失败'}")
    print(f"检测时间：{result['checked_at']}")
    for d in result.get("details", []):
        print(f"  {d}")
    for s in result.get("suggestions", []):
        print(f"  {s}")

"""Optional user-maintained institution preset loader.

The distributed schools.yaml is intentionally empty.
"""

from __future__ import annotations

from pathlib import Path
from typing import Any, Optional

try:
    import yaml  # type: ignore
except ImportError:
    yaml = None  # type: ignore

SCHOOLS_FILE = Path(__file__).resolve().parent.parent / "data" / "schools.yaml"


def _parse_scalar(value: str) -> Any:
    value = value.strip()
    if value in ("", "null", "None"):
        return None
    if value.startswith('"') and value.endswith('"'):
        return value[1:-1]
    if value.startswith("[") and value.endswith("]"):
        inner = value[1:-1].strip()
        if not inner:
            return []
        return [part.strip().strip('"') for part in inner.split(",")]
    return value


def _load_schools_without_yaml(text: str) -> list[dict[str, Any]]:
    """读取本仓库固定格式的 schools.yaml，避免 PyYAML 缺失时预设库失效。"""
    schools: list[dict[str, Any]] = []
    current: Optional[dict[str, Any]] = None
    section: Optional[str] = None

    for raw in text.splitlines():
        stripped = raw.strip()
        if not stripped or stripped.startswith("#") or stripped == "schools:":
            continue
        indent = len(raw) - len(raw.lstrip(" "))
        if stripped.startswith("- name:"):
            if current:
                schools.append(current)
            current = {"name": _parse_scalar(stripped.split(":", 1)[1]), "auth": {}}
            section = None
            continue
        if current is None:
            continue
        if indent == 4 and stripped.endswith(":"):
            section = stripped[:-1]
            current.setdefault(section, {})
            continue
        if ":" not in stripped:
            continue

        key, value = stripped.split(":", 1)
        if section and indent >= 6:
            current.setdefault(section, {})[key.strip()] = _parse_scalar(value)
        else:
            section = None
            current[key.strip()] = _parse_scalar(value)

    if current:
        schools.append(current)
    return schools


def load_schools() -> list[dict[str, Any]]:
    """加载预设学校库。优先用 PyYAML，缺失时使用内置简易解析器。"""
    if not SCHOOLS_FILE.exists():
        return []
    text = SCHOOLS_FILE.read_text(encoding="utf-8")
    if yaml is None:
        return _load_schools_without_yaml(text)
    data = yaml.safe_load(text)
    return data.get("schools", []) if data else []


def match_school(query: str) -> Optional[dict[str, Any]]:
    """模糊匹配学校。

    匹配顺序：精确名称 → 别名精确 → 包含匹配。
    返回匹配到的学校字典，未匹配返回 None。
    """
    query = query.strip().lower()
    if not query:
        return None

    schools = load_schools()
    if not schools:
        return None

    # 1. 精确匹配 name
    for s in schools:
        if s["name"].lower() == query:
            return s

    # 2. 别名精确匹配
    for s in schools:
        for alias in s.get("aliases", []):
            if alias.lower() == query:
                return s

    # 3. name 包含匹配
    for s in schools:
        if query in s["name"].lower():
            return s

    # 4. 别名包含匹配
    for s in schools:
        for alias in s.get("aliases", []):
            if query in alias.lower():
                return s

    return None


def list_school_names() -> list[str]:
    """返回所有预设学校名称列表。"""
    return [s["name"] for s in load_schools()]


if __name__ == "__main__":
    schools = load_schools()
    print(f"用户维护的机构预设数量：{len(schools)}")

"""字段校验模块。

对用户填写的配置字段做实时可达性探测：
- sso_domain: DNS 解析 + TCP 443 + HTTPS 证书
- carsi_entry: HTTP GET 探测
- ezproxy_url: HTTP GET 探测登录表单

所有校验函数返回 (ok: bool, message: str)。
"""

from __future__ import annotations

import socket
import ssl
import urllib.parse
import urllib.request


def validate_sso_domain(domain: str, timeout: float = 5.0) -> tuple[bool, str]:
    """校验 SSO 域名：DNS 解析 + TCP 443 + HTTPS 证书。

    返回 (是否通过, 说明消息)。
    """
    domain = domain.strip().lower()
    if not domain:
        return False, "域名为空"

    # 去掉协议前缀
    for prefix in ("https://", "http://"):
        if domain.startswith(prefix):
            domain = domain[len(prefix):]
    domain = domain.split("/")[0]

    # DNS 解析
    try:
        addrs = socket.getaddrinfo(domain, 443, socket.AF_UNSPEC, socket.SOCK_STREAM)
    except socket.gaierror:
        return False, f"DNS 解析失败：{domain}，请检查域名拼写"

    # TCP 443 连接 + TLS 握手
    last_err = ""
    for family, socktype, proto, _, sockaddr in addrs:
        try:
            with socket.create_connection(sockaddr, timeout=timeout) as sock:
                ctx = ssl.create_default_context()
                with ctx.wrap_socket(sock, server_hostname=domain) as ssock:
                    cert = ssock.getpeercert()
                    if cert is None:
                        return False, f"HTTPS 证书无效：{domain}"
            return True, f"SSO 域名可达：https://{domain}"
        except (socket.timeout, ConnectionRefusedError, OSError) as e:
            last_err = str(e)
            continue

    return False, f"无法连接到 https://{domain}（443 端口）：{last_err}"


def validate_carsi_entry(url: str, timeout: float = 8.0) -> tuple[bool, str]:
    """校验 CARSI 入口：HTTP GET 探测。

    判断标准：返回 2xx/3xx，且页面内容含「CARSI」「Shibboleth」或 SSO 跳转特征。
    """
    url = url.strip()
    if not url:
        return False, "CARSI 入口 URL 为空"

    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    try:
        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": "Mozilla/5.0 (lit-dl-config-validator)",
                "Accept": "text/html",
            },
        )
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            status = resp.getcode()
            body = resp.read(4096).decode("utf-8", errors="ignore").lower()

            if status >= 400:
                return False, f"CARSI 入口返回 HTTP {status}"

            # 内容特征检查
            keywords = ["carsi", "shibboleth", "idp", "sso", "login", "登录", "统一身份"]
            matched = [k for k in keywords if k in body]
            if matched:
                return True, f"CARSI 入口可达，检测到特征：{', '.join(matched[:3])}"

            # 3xx 跳转也算通过（可能跳到 SSO）
            if 300 <= status < 400:
                location = resp.headers.get("Location", "")
                return True, f"CARSI 入口跳转到：{location}"

            return True, f"CARSI 入口可达（HTTP {status}），但未检测到明显 SSO 特征"

    except urllib.error.URLError as e:
        return False, f"CARSI 入口不可达：{e.reason}"
    except Exception as e:
        return False, f"CARSI 入口探测异常：{e}"


def validate_ezproxy_url(url: str, timeout: float = 8.0) -> tuple[bool, str]:
    """校验 EZproxy 地址：HTTP GET 探测登录表单。

    判断标准：页面含 password 输入框。
    """
    url = url.strip()
    if not url:
        return False, "EZproxy URL 为空"

    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    try:
        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": "Mozilla/5.0 (lit-dl-config-validator)",
                "Accept": "text/html",
            },
        )
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            status = resp.getcode()
            body = resp.read(8192).decode("utf-8", errors="ignore").lower()

            if status >= 400:
                return False, f"EZproxy 返回 HTTP {status}"

            if "password" in body or "密码" in body:
                return True, "EZproxy 登录页可达，检测到密码输入框"

            return True, f"EZproxy 页面可达（HTTP {status}），但未检测到登录表单"

    except urllib.error.URLError as e:
        return False, f"EZproxy 不可达：{e.reason}"
    except Exception as e:
        return False, f"EZproxy 探测异常：{e}"


def validate_school_name(name: str) -> tuple[bool, str]:
    """校验学校名称基本格式。"""
    name = name.strip()
    if len(name) < 2:
        return False, "学校名称太短"
    if len(name) > 100:
        return False, "学校名称太长"
    return True, name


def validate_libraries(libraries: list[str]) -> tuple[bool, str]:
    """校验数据库清单。"""
    if not libraries:
        return False, "数据库清单不能为空"
    if len(libraries) > 50:
        return False, "数据库清单过长"
    return True, f"已选择 {len(libraries)} 个数据库"


# 已知数据库清单（用于向导多选提示）
KNOWN_DATABASES = [
    "知网 (CNKI)",
    "万方",
    "维普",
    "Web of Science",
    "Scopus",
    "IEEE Xplore",
    "ScienceDirect",
    "Springer Link",
    "Wiley Online Library",
    "ACS Publications",
    "RSC Publishing",
    "Nature",
    "Science",
    "Elsevier ScienceDirect",
    "Taylor & Francis",
    "SAGE Journals",
    "EBSCO",
    "ProQuest",
    "JSTOR",
    "中国知网",
]


if __name__ == "__main__":
    # 自检示例
    print("=== SSO 域名校验 ===")
    ok, msg = validate_sso_domain("login.university.example")
    print(f"  {ok}: {msg}")

    print("\n=== CARSI 入口校验 ===")
    ok, msg = validate_carsi_entry("https://www.carsi.edu.cn/")
    print(f"  {ok}: {msg}")

"""配置向导模块。

7 步交互式配置流程，供 AI 调用。
每一步返回一个 prompt 给用户，并接收用户输入进行校验。

使用方式（AI 调用）：
    from wizard import Wizard
    w = Wizard()
    # Step 1
    prompt = w.start()                # 返回给用户的提问
    # 用户回答后
    result = w.handle_step1(user_input)
    # result = {"next": "step2"|"retry"|"done", "prompt": str, "data": dict}

发行包不包含任何学校预设；机构名称和入口仅由用户在运行时提供。
"""

from __future__ import annotations

from datetime import datetime
from typing import Any, Optional
from urllib.parse import parse_qs, urlparse

from config import save_config, backup_config, delete_config, CONFIG_FILE
from schools_loader import match_school, list_school_names
from validators import (
    KNOWN_DATABASES,
    validate_carsi_entry,
    validate_ezproxy_url,
    validate_libraries,
    validate_school_name,
    validate_sso_domain,
)


def infer_access_from_url(url: str) -> dict[str, Any]:
    """Infer the likely library access route from a user-provided resource URL."""
    raw_url = url.strip()
    parsed = urlparse(raw_url if "://" in raw_url else f"https://{raw_url}")
    host = parsed.netloc.lower()
    path = parsed.path.lower()
    query = parse_qs(parsed.query)
    service = query.get("service", [""])[0]
    service_host = urlparse(service).netloc.lower() if service else ""

    result: dict[str, Any] = {
        "resource_entry": raw_url,
        "entry_host": host,
        "entry_type": "resource_entry",
        "auth_type": "custom",
        "sso_domain": host,
        "service_host": service_host or None,
        "institution_hint": None,
        "notes": "",
    }

    if "metaersp" in host or "metaauth" in host:
        result.update(
            {
                "entry_type": "resource_portal",
                "auth_type": "cas",
                "sso_domain": host,
                "institution_hint": host.split(".", 1)[0] if "." in host else None,
                "notes": "资源聚合门户；先从该入口进入，必要时由门户跳转到统一身份认证。",
            }
        )
    elif "/authserver/login" in path or host.startswith("cas."):
        hint = None
        if service_host:
            service_parts = urlparse(service).path.strip("/").split("/")
            if len(service_parts) >= 2:
                hint = service_parts[1]
        result.update(
            {
                "entry_type": "cas_login",
                "auth_type": "cas",
                "sso_domain": host,
                "institution_hint": hint,
                "notes": "CAS 登录入口；若 service 指向资源聚合平台，后续应回到该平台继续进入数据库。",
            }
        )
    elif "ezproxy" in host or "libproxy" in host:
        result.update({"entry_type": "ezproxy", "auth_type": "custom", "notes": "图书馆远程访问代理入口。"})
    elif "webvpn" in host or "vpn" in host:
        result.update({"entry_type": "webvpn", "auth_type": "custom", "notes": "WebVPN 入口。"})
    elif "shibboleth" in path or "carsi" in host:
        result.update({"entry_type": "carsi", "auth_type": "sso", "notes": "CARSI/Shibboleth 机构认证入口。"})

    return result


class Wizard:
    """配置向导状态机。

    状态流转：
        step1 (学校名) ->
            命中预设 -> step4 (数据库确认) -> step6 (自检) -> step7 (保存)
            未命中 -> step2 (CARSI自查) -> step3 (SSO域名) -> step4 -> step6 -> step7
                                                   或 step5 (EZproxy) -> step6 -> step7
    """

    def __init__(self) -> None:
        self.state: str = "step1"
        self.data: dict[str, Any] = {}
        self.matched_preset: Optional[dict[str, Any]] = None

    # ===== Step 1: 询问学校名称 =====
    def start(self) -> str:
        """返回首步提问。"""
        self.state = "step1"
        return (
            "你好！我是文献下载助手。首次使用需要先配置你的图书馆资源入口（只需一次）。\n\n"
            "请先发你平时进入图书馆电子资源/数据库的平台链接。\n"
            "可以是资源门户、数据库列表、Web of Science 入口、某个数据库详情页，"
            "或跳转到统一身份认证的登录链接。\n\n"
            "我会先根据链接判断 CAS/CARSI/EZproxy/WebVPN/聚合门户等授权路径；"
            "发行包不内置学校名称或认证地址。"
        )

    def handle_step1(self, user_input: str) -> dict[str, Any]:
        """处理资源入口链接；非 URL 输入按旧的学校名预设兜底。"""
        value = user_input.strip()
        if not value:
            return {"next": "retry", "prompt": "输入不能为空，请粘贴图书馆电子资源或数据库入口链接："}

        if "://" in value or "." in value:
            inferred = infer_access_from_url(value)
            self.data.update(inferred)
            self.data["school_name"] = inferred.get("institution_hint") or inferred["entry_host"]
            self.data["source"] = "resource_url"
            self.data["auth_type"] = inferred["auth_type"]
            self.data["sso_domain"] = inferred["sso_domain"]
            self.data["carsi_entry"] = inferred["resource_entry"]
            self.data["libraries"] = ["Web of Science", "ScienceDirect", "Springer", "IEEE Xplore", "知网", "ACS"]
            self.data["notes"] = inferred.get("notes", "")
            self.data["discovery"] = {"resource_entry_url": inferred["resource_entry"]}
            if inferred["entry_type"] == "resource_portal":
                self.data["discovery"]["resource_portal_url"] = inferred["resource_entry"]
            if inferred.get("service_host"):
                self.data["discovery"]["auth_service_host"] = inferred["service_host"]

            self.state = "step4"
            return {
                "next": "step4",
                "prompt": (
                    "已根据资源链接识别授权路径：\n"
                    f"  入口类型：{inferred['entry_type']}\n"
                    f"  认证类型：{inferred['auth_type']}\n"
                    f"  SSO 域名：{inferred['sso_domain']}\n"
                    f"  资源入口：{inferred['resource_entry']}\n\n"
                    "确认先使用这组配置吗？\n"
                    "  1. 确认，继续自检\n"
                    "  2. 我想调整数据库清单\n"
                    "  3. 改用手动配置"
                ),
                "data": {"inferred": inferred},
            }

        name = value

        ok, msg = validate_school_name(name)
        if not ok:
            return {"next": "retry", "prompt": f"{msg}，请重新输入资源链接或学校名称："}

        # 查预设库
        preset = match_school(name)
        if preset:
            self.matched_preset = preset
            self.data["school_name"] = preset["name"]
            self.data["source"] = "preset"

            # 预填预设库的配置
            auth = preset.get("auth", {})
            self.data["auth_type"] = auth.get("type", "cas")
            self.data["sso_domain"] = auth.get("sso_domain", "")
            self.data["carsi_entry"] = auth.get("carsi_entry", "")
            self.data["libraries"] = preset.get("libraries", [])
            self.data["notes"] = preset.get("notes", "")

            self.state = "step4"
            return {
                "next": "step4",
                "prompt": (
                    f"已匹配到预设学校：{preset['name']}\n"
                    f"  认证类型：{auth.get('type', '未知')}\n"
                    f"  SSO 域名：{auth.get('sso_domain', '未知')}\n"
                    f"  CARSI 入口：{auth.get('carsi_entry', '未配置')}\n"
                    f"  预设数据库：{', '.join(preset.get('libraries', []))}\n\n"
                    "确认使用以上配置吗？\n"
                    "  1. 确认，直接完成配置\n"
                    "  2. 我想调整数据库清单\n"
                    "  3. 这不是我的学校，重新输入"
                ),
                "data": {"matched": preset["name"]},
            }

        # 未命中预设，进入自助填写
        self.data["school_name"] = name
        self.data["source"] = "manual"
        self.state = "step2"
        return {
            "next": "step2",
            "prompt": (
                f"未在预设库中匹配到「{name}」，进入自助配置向导。\n\n"
                "Step 2: 你的学校/单位是否接入 CARSI 联邦认证？\n"
                "（CARSI 是高校统一身份认证联邦，查询入口：https://www.carsi.edu.cn/）\n\n"
                "  1. 是，已接入 CARSI\n"
                "  2. 否 / 不清楚，走图书馆远程访问（EZproxy）\n"
                "  3. 都没有，我用 VPN 连校园网"
            ),
        }

    # ===== Step 2: CARSI 自查 =====
    def handle_step2(self, user_input: str) -> dict[str, Any]:
        """处理 CARSI 选择。"""
        choice = user_input.strip()
        if choice == "1":
            self.data["use_carsi"] = True
            self.state = "step2b"
            return {
                "next": "step2b",
                "prompt": (
                    "请粘贴贵校的 CARSI 入口 URL。\n"
                    "（可在 https://www.carsi.edu.cn/ 成员机构列表中找到，"
                    "例如 https://login.university.example/idp/shibboleth；请使用本机构实际入口）\n\n"
                    "如果找不到，输入「跳过」留空，后续可手动补充。"
                ),
            }
        elif choice == "2":
            self.data["use_carsi"] = False
            self.state = "step3"
            return {
                "next": "step3",
                "prompt": (
                    "你选择了图书馆远程访问（EZproxy）方式。\n\n"
                    "Step 3: 请输入贵校统一身份认证域名。\n"
                    "（例如 login.university.example；请以本机构实际页面为准）"
                ),
            }
        elif choice == "3":
            self.data["use_carsi"] = False
            self.data["use_vpn"] = True
            self.state = "step3"
            return {
                "next": "step3",
                "prompt": (
                    "好的，使用 VPN 方式。\n"
                    "请先确保已连接校园 VPN，然后输入贵校统一身份认证域名。\n"
                    "（例如 login.university.example；请使用本机构实际域名）"
                ),
            }
        return {"next": "retry", "prompt": "请输入 1、2 或 3："}

    # ===== Step 2b: CARSI 入口 URL =====
    def handle_step2b(self, user_input: str) -> dict[str, Any]:
        """处理 CARSI 入口 URL 输入。"""
        url = user_input.strip()
        if url in ("跳过", "skip", ""):
            self.data["carsi_entry"] = ""
            self.state = "step3"
            return {
                "next": "step3",
                "prompt": (
                    "已跳过 CARSI 入口（后续可补充）。\n\n"
                    "Step 3: 请输入贵校统一身份认证域名。\n"
                    "（例如 login.university.example；请使用本机构实际域名）"
                ),
            }

        # 校验
        ok, msg = validate_carsi_entry(url)
        if not ok:
            return {
                "next": "retry",
                "prompt": f"{msg}\n\n请重新输入 CARSI 入口 URL，或输入「跳过」留空：",
            }

        self.data["carsi_entry"] = url
        self.state = "step3"
        return {
            "next": "step3",
            "prompt": (
                f"CARSI 入口校验通过。\n\n"
                "Step 3: 请输入贵校统一身份认证域名。\n"
                "（例如 login.university.example；请使用本机构实际域名）"
            ),
        }

    # ===== Step 3: SSO 域名 =====
    def handle_step3(self, user_input: str) -> dict[str, Any]:
        """处理 SSO 域名输入。"""
        domain = user_input.strip()
        if not domain:
            return {"next": "retry", "prompt": "域名不能为空，请重新输入："}

        ok, msg = validate_sso_domain(domain)
        if not ok:
            return {
                "next": "retry",
                "prompt": f"{msg}\n\n请重新输入本机构的 SSO 域名（如 login.university.example）：",
            }

        self.data["sso_domain"] = domain.split("://")[-1].split("/")[0]
        # 如果还没设 auth_type，默认 cas
        if "auth_type" not in self.data:
            self.data["auth_type"] = "cas"

        # 如果走 EZproxy 路径
        if not self.data.get("use_carsi", True) and not self.data.get("use_vpn"):
            self.state = "step5"
            return {
                "next": "step5",
                "prompt": (
                    f"SSO 域名校验通过：{msg}\n\n"
                    "Step 5: 请输入贵校图书馆 EZproxy 登录地址。\n"
                    "（提示：使用 EZproxy 前通常需先在图书馆网站开通「远程访问」权限）\n\n"
                    "如果不确定，输入「跳过」可稍后补充。"
                ),
            }

        self.state = "step4"
        return {
            "next": "step4",
            "prompt": (
                f"SSO 域名校验通过：{msg}\n\n"
                "Step 4: 你常下载的数据库有哪些？\n"
                f"可选：{', '.join(KNOWN_DATABASES[:10])} ...\n\n"
                "请输入数据库名称，多个用逗号或空格分隔："
            ),
        }

    # ===== Step 4: 数据库多选 =====
    def handle_step4(self, user_input: str) -> dict[str, Any]:
        """处理数据库选择。"""
        if user_input.strip() in ("确认", "1", "ok", "yes", "好"):
            # 预设学校确认流程
            if not self.data.get("libraries"):
                return {"next": "retry", "prompt": "请输入数据库名称："}
        else:
            # 解析输入
            libs = [s.strip() for s in user_input.replace(",", " ").split() if s.strip()]
            if libs:
                self.data["libraries"] = libs

        ok, msg = validate_libraries(self.data.get("libraries", []))
        if not ok:
            return {"next": "retry", "prompt": f"{msg}，请重新输入："}

        self.state = "step6"
        return {
            "next": "step6",
            "prompt": (
                f"已记录 {len(self.data['libraries'])} 个数据库。\n\n"
                "Step 6: 正在进行连通性自检..."
            ),
        }

    # ===== Step 5: EZproxy 地址 =====
    def handle_step5(self, user_input: str) -> dict[str, Any]:
        """处理 EZproxy 地址输入。"""
        url = user_input.strip()
        if url in ("跳过", "skip", ""):
            self.data["ezproxy_url"] = None
            self.state = "step4"
            return {
                "next": "step4",
                "prompt": (
                    "已跳过 EZproxy。\n\n"
                    "Step 4: 你常下载的数据库有哪些？\n"
                    "请输入数据库名称，多个用逗号或空格分隔："
                ),
            }

        ok, msg = validate_ezproxy_url(url)
        if not ok:
            return {
                "next": "retry",
                "prompt": f"{msg}\n\n请重新输入 EZproxy 地址，或输入「跳过」：",
            }

        self.data["ezproxy_url"] = url
        self.state = "step4"
        return {
            "next": "step4",
            "prompt": (
                f"EZproxy 校验通过：{msg}\n\n"
                "Step 4: 你常下载的数据库有哪些？\n"
                "请输入数据库名称，多个用逗号或空格分隔："
            ),
        }

    # ===== Step 6: 连通性自检 =====
    def handle_step6(self, user_input: str = "") -> dict[str, Any]:
        """执行连通性自检。"""
        # 延迟导入避免循环依赖
        from health_check import health_check, clear_cache

        # 先保存临时配置再做自检
        try:
            temp_config = self._build_config()
        except ValueError as e:
            return {"next": "retry", "prompt": f"配置构建失败：{e}"}

        # 临时保存用于自检
        clear_cache()
        save_config(temp_config)
        result = health_check(force=True)

        self.state = "step7"
        details_text = "\n".join(f"  {d}" for d in result.get("details", []))
        if result["ok"]:
            return {
                "next": "step7",
                "prompt": (
                    f"连通性自检通过：\n{details_text}\n\n"
                    "Step 7: 确认保存配置吗？\n  1. 确认保存\n  2. 重新配置"
                ),
            }
        else:
            suggestions = "\n".join(f"  {s}" for s in result.get("suggestions", []))
            return {
                "next": "step7",
                "prompt": (
                    f"连通性自检未完全通过：\n{details_text}\n\n"
                    f"{suggestions}\n\n"
                    "Step 7: 如何处理？\n"
                    "  1. 仍然保存（可后续修正）\n"
                    "  2. 重新配置"
                ),
                "data": {"warnings": result.get("details", [])},
            }

    # ===== Step 7: 持久化 =====
    def handle_step7(self, user_input: str) -> dict[str, Any]:
        """处理最终保存确认。"""
        choice = user_input.strip()
        if choice in ("2", "重新配置", "重新"):
            delete_config()
            self.__init__()
            return {"next": "step1", "prompt": self.start()}

        # 保存
        try:
            config = self._build_config()
            # 附加警告（如有）
            if self.data.get("warnings"):
                config["_warnings"] = self.data["warnings"]
            path = save_config(config)
            return {
                "next": "done",
                "prompt": (
                    f"已为「{self.data['school_name']}」完成配置！\n"
                    f"配置文件：{path}\n\n"
                    "现在可以直接下载文献了。如需切换学校，随时说「换学校」或「/reconfig」。"
                ),
                "data": {"school": self.data["school_name"], "path": str(path)},
            }
        except ValueError as e:
            return {"next": "retry", "prompt": f"保存失败：{e}\n请检查后重试："}

    # ===== 调度入口 =====
    def handle(self, user_input: str) -> dict[str, Any]:
        """根据当前状态调度到对应处理函数。"""
        handlers = {
            "step1": self.handle_step1,
            "step2": self.handle_step2,
            "step2b": self.handle_step2b,
            "step3": self.handle_step3,
            "step4": self.handle_step4,
            "step5": self.handle_step5,
            "step6": self.handle_step6,
            "step7": self.handle_step7,
        }
        handler = handlers.get(self.state)
        if handler is None:
            self.state = "step1"
            return {"next": "step1", "prompt": self.start()}
        return handler(user_input)

    # ===== 非交互式：预设学校一键配置 =====
    def configure_from_preset(self, school_name: str) -> dict[str, Any]:
        """预设学校一键配置（非交互式）。

        成功返回保存后的配置字典，失败抛 ValueError。
        """
        preset = match_school(school_name)
        if not preset:
            raise ValueError(f"预设库中未找到「{school_name}」")

        auth = preset.get("auth", {})
        config = {
            "version": 1,
            "school": {
                "name": preset["name"],
                "code": preset.get("aliases", [""])[0] if preset.get("aliases") else None,
                "configured_at": datetime.now().isoformat(),
                "source": "preset",
            },
            "auth": {
                "type": auth.get("type", "cas"),
                "sso_domain": auth.get("sso_domain", ""),
                "carsi_entry": auth.get("carsi_entry") or None,
                "carsi_sp_entity_id": None,
            },
            "proxy": {
                "type": None,
                "ezproxy_url": None,
            },
            "libraries": preset.get("libraries", []),
            "discovery": preset.get("discovery", {}),
            "notes": preset.get("notes", ""),
        }

        path = save_config(config)
        return {"config": config, "path": str(path)}

    # ===== 非交互式：资源入口链接配置 =====
    def configure_from_resource_url(self, resource_url: str) -> dict[str, Any]:
        """Configure from a library resource portal or authentication URL."""
        inferred = infer_access_from_url(resource_url)
        self.data.update(inferred)
        self.data["school_name"] = inferred.get("institution_hint") or inferred["entry_host"]
        self.data["source"] = "resource_url"
        self.data["auth_type"] = inferred["auth_type"]
        self.data["sso_domain"] = inferred["sso_domain"]
        self.data["carsi_entry"] = inferred["resource_entry"]
        self.data["libraries"] = ["Web of Science", "ScienceDirect", "Springer", "IEEE Xplore", "知网", "ACS"]
        self.data["notes"] = inferred.get("notes", "")
        self.data["discovery"] = {"resource_entry_url": inferred["resource_entry"]}
        if inferred["entry_type"] == "resource_portal":
            self.data["discovery"]["resource_portal_url"] = inferred["resource_entry"]
        if inferred.get("service_host"):
            self.data["discovery"]["auth_service_host"] = inferred["service_host"]

        config = self._build_config()
        path = save_config(config)
        return {"config": config, "path": str(path), "inferred": inferred}

    # ===== 内部：构建配置字典 =====
    def _build_config(self) -> dict[str, Any]:
        """从向导收集的数据构建配置字典。"""
        if not self.data.get("school_name"):
            raise ValueError("学校名称未设置")
        if not self.data.get("sso_domain"):
            raise ValueError("SSO 域名未设置")
        if not self.data.get("libraries"):
            raise ValueError("数据库清单未设置")

        return {
            "version": 1,
            "school": {
                "name": self.data["school_name"],
                "code": None,
                "configured_at": datetime.now().isoformat(),
                "source": self.data.get("source", "manual"),
            },
            "auth": {
                "type": self.data.get("auth_type", "cas"),
                "sso_domain": self.data["sso_domain"],
                "carsi_entry": self.data.get("carsi_entry") or None,
                "carsi_sp_entity_id": None,
            },
            "proxy": {
                "type": "ezproxy" if self.data.get("ezproxy_url") else None,
                "ezproxy_url": self.data.get("ezproxy_url") or None,
            },
            "libraries": self.data["libraries"],
            "discovery": self.data.get("discovery", {}),
            "notes": self.data.get("notes", ""),
        }


if __name__ == "__main__":
    # 非交互式一键配置示例
    w = Wizard()
    try:
        result = w.configure_from_preset("交大")
        print(f"配置成功：{result['config']['school']['name']}")
        print(f"配置文件：{result['path']}")
    except ValueError as e:
        print(f"配置失败：{e}")
        print(f"配置文件路径：{CONFIG_FILE}")

import json
import os
import stat
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "configure_credentials.py"


class ConfigureCredentialsTest(unittest.TestCase):
    def test_cli_stdin_saves_key_without_echoing_it(self):
        secret = "publisher-secret-12345678"
        with tempfile.TemporaryDirectory() as tmp:
            env = os.environ.copy()
            env["LIT_DL_CONFIG_DIR"] = tmp
            result = subprocess.run(
                [sys.executable, str(SCRIPT), "set", "elsevier", "--stdin"],
                cwd=ROOT,
                env=env,
                input=secret + "\n",
                text=True,
                capture_output=True,
            )
            credentials_path = Path(tmp) / "credentials.json"
            saved = json.loads(credentials_path.read_text(encoding="utf-8"))
            mode = stat.S_IMODE(credentials_path.stat().st_mode)

        self.assertEqual(result.returncode, 0)
        self.assertNotIn(secret, result.stdout)
        self.assertNotIn(secret, result.stderr)
        self.assertEqual(saved["elsevier"]["api_key"], secret)
        self.assertEqual(mode, 0o600)


if __name__ == "__main__":
    unittest.main()

import importlib
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
SRC = ROOT / "src"
SCRIPT = ROOT / "scripts" / "configure_school.py"


class ConfigWizardTest(unittest.TestCase):
    def test_start_asks_for_library_resource_url_first(self):
        sys.path.insert(0, str(SRC))
        try:
            wizard = importlib.import_module("wizard")
            wizard = importlib.reload(wizard)
            prompt = wizard.Wizard().start()
        finally:
            if str(SRC) in sys.path:
                sys.path.remove(str(SRC))

        self.assertIn("图书馆", prompt)
        self.assertIn("资源", prompt)
        self.assertIn("链接", prompt)
        self.assertNotIn("请问你所在的学校或单位是", prompt)

    def test_infer_generic_resource_portal_from_resource_url(self):
        sys.path.insert(0, str(SRC))
        try:
            wizard = importlib.import_module("wizard")
            wizard = importlib.reload(wizard)
            result = wizard.infer_access_from_url("https://portal.metaersp.example/personal")
        finally:
            if str(SRC) in sys.path:
                sys.path.remove(str(SRC))

        self.assertEqual(result["entry_type"], "resource_portal")
        self.assertEqual(result["auth_type"], "cas")
        self.assertEqual(result["sso_domain"], "portal.metaersp.example")
        self.assertEqual(result["resource_entry"], "https://portal.metaersp.example/personal")
        self.assertEqual(result["institution_hint"], "portal")

    def test_infer_cas_login_service_callback_from_resource_url(self):
        sys.path.insert(0, str(SRC))
        try:
            wizard = importlib.import_module("wizard")
            wizard = importlib.reload(wizard)
            result = wizard.infer_access_from_url(
                "https://login.university.example/authserver/login?service=https%3A%2F%2Fresources.university.example%2Fservice%2Fcampus%2Fcallback"
            )
        finally:
            if str(SRC) in sys.path:
                sys.path.remove(str(SRC))

        self.assertEqual(result["entry_type"], "cas_login")
        self.assertEqual(result["auth_type"], "cas")
        self.assertEqual(result["sso_domain"], "login.university.example")
        self.assertEqual(result["service_host"], "resources.university.example")
        self.assertEqual(result["institution_hint"], "campus")

    def test_resource_url_flow_can_save_schema_valid_config(self):
        with tempfile.TemporaryDirectory() as tmp:
            old_env = os.environ.copy()
            os.environ["LIT_DL_CONFIG_DIR"] = tmp
            sys.path.insert(0, str(SRC))
            try:
                config = importlib.import_module("config")
                wizard = importlib.import_module("wizard")
                config = importlib.reload(config)
                wizard = importlib.reload(wizard)
                w = wizard.Wizard()
                w.handle_step1("https://portal.metaersp.example/personal")
                result = w.handle_step7("1")

                self.assertEqual(result["next"], "done")
                saved = Path(result["data"]["path"])
                data = json.loads(saved.read_text(encoding="utf-8"))
                self.assertEqual(data["school"]["source"], "resource_url")
                self.assertEqual(data["auth"]["sso_domain"], "portal.metaersp.example")
                self.assertEqual(data["discovery"]["resource_portal_url"], "https://portal.metaersp.example/personal")
            finally:
                if str(SRC) in sys.path:
                    sys.path.remove(str(SRC))
                os.environ.clear()
                os.environ.update(old_env)

    def test_distributed_skill_has_no_institution_presets(self):
        sys.path.insert(0, str(SRC))
        try:
            schools_loader = importlib.import_module("schools_loader")
            schools_loader = importlib.reload(schools_loader)
            presets = schools_loader.load_schools()
        finally:
            if str(SRC) in sys.path:
                sys.path.remove(str(SRC))

        self.assertEqual(presets, [])
        self.assertIsNone(schools_loader.match_school("Example University"))

    def test_cli_show_reports_missing_config(self):
        with tempfile.TemporaryDirectory() as tmp:
            env = os.environ.copy()
            env["LIT_DL_CONFIG_DIR"] = tmp
            result = subprocess.run(
                [sys.executable, str(SCRIPT), "show"],
                cwd=ROOT,
                env=env,
                text=True,
                capture_output=True,
            )

        self.assertEqual(result.returncode, 2)
        self.assertIn("尚未配置", result.stdout)

    def test_cli_url_configures_from_resource_entry(self):
        with tempfile.TemporaryDirectory() as tmp:
            env = os.environ.copy()
            env["LIT_DL_CONFIG_DIR"] = tmp
            result = subprocess.run(
                [sys.executable, str(SCRIPT), "url", "https://portal.metaersp.example/personal"],
                cwd=ROOT,
                env=env,
                text=True,
                capture_output=True,
            )

        self.assertEqual(result.returncode, 0, result.stderr)
        data = json.loads(result.stdout)
        self.assertTrue(data["ok"])
        self.assertEqual(data["entry_type"], "resource_portal")
        self.assertEqual(data["sso_domain"], "portal.metaersp.example")

    def test_cli_infer_does_not_save_config(self):
        with tempfile.TemporaryDirectory() as tmp:
            env = os.environ.copy()
            env["LIT_DL_CONFIG_DIR"] = tmp
            result = subprocess.run(
                [sys.executable, str(SCRIPT), "infer", "https://portal.metaersp.example/personal"],
                cwd=ROOT,
                env=env,
                text=True,
                capture_output=True,
            )

            self.assertFalse((Path(tmp) / "school.json").exists())

        self.assertEqual(result.returncode, 0, result.stderr)
        data = json.loads(result.stdout)
        self.assertEqual(data["entry_type"], "resource_portal")

    def test_cli_cnki_url_updates_existing_config(self):
        with tempfile.TemporaryDirectory() as tmp:
            env = os.environ.copy()
            env["LIT_DL_CONFIG_DIR"] = tmp
            configured = subprocess.run(
                [sys.executable, str(SCRIPT), "url", "https://portal.metaersp.example/personal"],
                cwd=ROOT,
                env=env,
                text=True,
                capture_output=True,
            )
            self.assertEqual(configured.returncode, 0, configured.stderr)

            result = subprocess.run(
                [sys.executable, str(SCRIPT), "cnki-url", "https://kns.cnki.net/kns8s/defaultresult/index"],
                cwd=ROOT,
                env=env,
                text=True,
                capture_output=True,
            )

            self.assertEqual(result.returncode, 0, result.stderr)
            data = json.loads(result.stdout)
            self.assertTrue(data["ok"])
            saved = json.loads((Path(tmp) / "school.json").read_text(encoding="utf-8"))
            self.assertEqual(saved["discovery"]["cnki_url"], "https://kns.cnki.net/kns8s/defaultresult/index")


if __name__ == "__main__":
    unittest.main()

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs, selectBatchDois } from "../../scripts/batch_download.mjs";

describe("batch CLI contract", () => {
  test("does not create an output directory before SI is confirmed", () => {
    const parent = fs.mkdtempSync(path.join(os.tmpdir(), "si-gate-"));
    const out = path.join(parent, "must-not-exist");
    const script = fileURLToPath(new URL("../../scripts/batch_download.mjs", import.meta.url));
    const run = spawnSync(process.execPath, [script, "--title", "Example", "--out", out], { encoding: "utf8" });
    assert.equal(run.status, 2);
    assert.match(run.stdout, /si_confirmation_required/);
    assert.equal(fs.existsSync(out), false);
  });

  test("rejects conflicting SI choices", () => {
    assert.throws(
      () => parseArgs(["node", "batch_download.mjs", "--title", "Example", "--si", "--no-si"]),
      /mutually exclusive/i
    );
  });

  test("accepts publisher-scoped web fallback decisions", () => {
    const args = parseArgs([
      "node", "batch_download.mjs", "--dois", "10.1016/example", "--no-si",
      "--api-fallback-web-for", "elsevier,ieee",
      "--no-api-fallback-web-for", "springer_nature",
    ]);
    assert.deepEqual(args.apiFallbackWebFor, ["elsevier", "ieee"]);
    assert.deepEqual(args.noApiFallbackWebFor, ["springer_nature"]);
  });

  test("accepts a title alongside a known PDF URL", () => {
    const args = parseArgs([
      "node", "batch_download.mjs",
      "--pdf-url", "https://example.org/paper.pdf",
      "--title", "Example Paper",
      "--no-si",
    ]);
    assert.equal(args.pdfUrl, "https://example.org/paper.pdf");
    assert.equal(args.title, "Example Paper");
  });

  test("does not silently truncate an explicit DOI list", () => {
    const dois = Array.from({ length: 12 }, (_, index) => `10.1234/item-${index + 1}`);
    const args = parseArgs([
      "node", "batch_download.mjs", "--dois", dois.join(","), "--no-si",
    ]);
    assert.equal(args.count, undefined);
    assert.deepEqual(selectBatchDois(args.dois, args.count), dois);
  });

  test("keeps the topic default and honors an explicit DOI count", () => {
    const topic = parseArgs(["node", "batch_download.mjs", "--topic", "rice", "--no-si"]);
    assert.equal(topic.count, 10);
    assert.deepEqual(selectBatchDois(["a", "b", "c"], 2), ["a", "b"]);
  });

  test("requires a title for a zh non-CNKI PDF route without touching the network", () => {
    const parent = fs.mkdtempSync(path.join(os.tmpdir(), "cnki-title-required-"));
    const script = fileURLToPath(new URL("../../scripts/batch_download.mjs", import.meta.url));
    const run = spawnSync(process.execPath, [
      script,
      "--pdf-url", "https://example.org/paper.pdf",
      "--language", "zh",
      "--no-si",
      "--out", parent,
    ], { encoding: "utf8" });
    assert.equal(run.status, 0);
    assert.match(run.stdout, /metadata_ambiguous/);
    assert.match(run.stdout, /CNKI title search requires --title/);
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import {
  DEFAULT_CNKI_URL,
  cnkiSearchUrl,
  createCnkiTransport,
  filterCnkiDownloadCandidates,
  isCnkiUrl,
  looksChinese,
  legacyCnkiDetailUrl,
  safeCnkiFileName,
} from "../../scripts/lib/cnki.mjs";

describe("CNKI helpers", () => {
  test("detects Chinese titles", () => {
    assert.equal(looksChinese("乡村振兴背景下数字治理研究"), true);
    assert.equal(looksChinese("Digital governance in rural China"), false);
  });

  test("builds a CNKI search URL with kw", () => {
    const url = new URL(cnkiSearchUrl("乡村振兴", DEFAULT_CNKI_URL));
    assert.equal(url.hostname, "kns.cnki.net");
    assert.equal(url.searchParams.get("kw"), "乡村振兴");
  });

  test("recognizes CNKI hosts", () => {
    assert.equal(isCnkiUrl("https://kns.cnki.net/kcms/detail/detail.aspx"), true);
    assert.equal(isCnkiUrl("https://navi.cnki.com.cn/knavi/"), true);
    assert.equal(isCnkiUrl("https://example.org/paper"), false);
  });

  test("creates safe CNKI filenames", () => {
    assert.equal(
      safeCnkiFileName("乡村振兴: 数字治理/路径?", ".pdf"),
      "乡村振兴_数字治理路径.pdf"
    );
    assert.equal(safeCnkiFileName("", ".caj"), "cnki-paper.caj");
  });

  test("filters CNKI candidates to PDF-only when requested", () => {
    const candidates = [
      { text: "下载", url: "https://kns.cnki.net/kcms/download.aspx?filename=abc" },
      { text: "PDF下载", url: "https://kns.cnki.net/kcms/download.aspx?filename=abc&dflag=pdf" },
      { text: "CAJ下载", url: "https://kns.cnki.net/kcms/download.aspx?filename=abc&dflag=caj" },
    ];
    assert.deepEqual(filterCnkiDownloadCandidates(candidates, "pdf"), [candidates[1]]);
    assert.deepEqual(filterCnkiDownloadCandidates(candidates, "any"), candidates);
  });

  test("keeps browser/FSSO/WebVPN access behind one transport seam", () => {
    assert.equal(createCnkiTransport({ mode: "fsso" }).resolve("https://kns.cnki.net/a"), "https://kns.cnki.net/a");
    assert.equal(
      createCnkiTransport({ mode: "webvpn", webvpnUrlTemplate: "https://vpn.example/proxy?url={url}" }).resolve("https://kns.cnki.net/a"),
      "https://vpn.example/proxy?url=https%3A%2F%2Fkns.cnki.net%2Fa"
    );
  });

  test("builds the legacy CNKI detail fallback from strong identifiers", () => {
    const url = new URL(legacyCnkiDetailUrl({ filename: "ABC123", dbcode: "CJFD" }));
    assert.equal(url.pathname, "/kcms/detail/detail.aspx");
    assert.equal(url.searchParams.get("filename"), "ABC123");
    assert.equal(url.searchParams.get("dbcode"), "CJFD");
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
  credentialsPathFromEnv,
  deleteProviderCredentials,
  loadCredentials,
  maskSecret,
  saveProviderCredentials,
} from "../../scripts/lib/credentials.mjs";

describe("publisher credential storage", () => {
  test("stores secrets separately with owner-only permissions and masks output", () => {
    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lit-dl-creds-"));
    const env = { LIT_DL_CONFIG_DIR: dir };
    const file = credentialsPathFromEnv(env);

    saveProviderCredentials("elsevier", { api_key: "secret-12345678", insttoken: "inst-87654321" }, env);

    assert.equal(file, path.join(dir, "credentials.json"));
    assert.equal(fs.statSync(file).mode & 0o777, 0o600);
    assert.deepEqual(loadCredentials(env).elsevier, {
      api_key: "secret-12345678",
      insttoken: "inst-87654321",
    });
    assert.equal(maskSecret("secret-12345678"), "***********5678");

    deleteProviderCredentials("elsevier", env);
    assert.equal(loadCredentials(env).elsevier, undefined);
  });
});

import { test } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const runtimeRoots = ["SKILL.md", "README.md", "agents", "data", "examples", "scripts", "src"];
const textExtensions = new Set([".md", ".yaml", ".yml", ".json", ".py", ".mjs", ".tsv", ".txt"]);

function filesUnder(relative) {
  const target = path.join(root, relative);
  if (!fs.existsSync(target)) return [];
  if (fs.statSync(target).isFile()) return [target];
  return fs.readdirSync(target, { withFileTypes: true }).flatMap((entry) =>
    entry.name === "__pycache__" ? [] : filesUnder(path.join(relative, entry.name))
  );
}

test("distributed skill contains no local identity, institution preset, or secret file", () => {
  const files = runtimeRoots.flatMap(filesUnder).filter((file) => textExtensions.has(path.extname(file)));
  const joined = files.map((file) => `${path.relative(root, file)}\n${fs.readFileSync(file, "utf8")}`).join("\n");

  assert.doesNotMatch(joined, /\/Users\/[a-z0-9._-]+\/|C:\\Users\\[a-z0-9._-]+\\/i);
  const legacyInstitutionMarkers = new RegExp(
    ["s" + "jtu", "j" + "account", "w" + "hu\\."].join("|"),
    "i"
  );
  assert.doesNotMatch(joined, legacyInstitutionMarkers);
  assert.doesNotMatch(joined, /(?:id|cas|sso|passport|authserver)\.[a-z0-9-]+\.edu\.cn/i);

  const schools = fs.readFileSync(path.join(root, "data/schools.yaml"), "utf8");
  assert.match(schools, /^schools:\s*\[\]\s*$/m);

  for (const forbidden of ["credentials.json", "school.json", ".env"]) {
    assert.equal(files.some((file) => path.basename(file) === forbidden), false, `${forbidden} must not be distributed`);
  }
});

import { test } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { writeManifest } from "../../scripts/lib/manifest.mjs";

test("manifest records route and SI choice without secrets", () => {
  const outDir = fs.mkdtempSync(path.join(os.tmpdir(), "manifest-"));
  const file = writeManifest(outDir, {
    request: { si_requested: false },
    results: [{
      doi: "10.1016/example",
      provider: "elsevier",
      api_key: "must-not-leak",
      source: "https://api.example.test/fulltext?api_key=query-secret&doi=10.1016/example",
      reason: "Authorization: Bearer bearer-secret",
      status: "downloaded",
    }],
  });
  const text = fs.readFileSync(file, "utf8");
  assert.doesNotMatch(text, /must-not-leak/);
  assert.doesNotMatch(text, /query-secret|bearer-secret/);
  assert.match(text, /api_key=\[REDACTED\]/);
  assert.match(text, /"si_requested": false/);
  assert.match(text, /"provider": "elsevier"/);
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import { parseCrossrefWork } from "../../scripts/lib/metadata.mjs";

describe("metadata normalization", () => {
  test("normalizes a Crossref work into one literature entity", () => {
    const article = parseCrossrefWork({
      DOI: "10.1016/J.CELL.2026.01.001",
      title: ["A useful article"],
      publisher: "Elsevier BV",
      language: "en",
      license: [{ URL: "https://creativecommons.org/licenses/by/4.0/" }],
      link: [{ URL: "https://example.org/article.pdf", "content-type": "application/pdf", "intended-application": "text-mining" }],
    });
    assert.equal(article.doi, "10.1016/j.cell.2026.01.001");
    assert.equal(article.title, "A useful article");
    assert.equal(article.publisher, "Elsevier BV");
    assert.equal(article.language, "en");
    assert.equal(article.license, "https://creativecommons.org/licenses/by/4.0/");
    assert.equal(article.publisherPdfUrl, "https://example.org/article.pdf");
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { downloadOpenAccessArticle } from "../../scripts/lib/open-access-provider.mjs";

describe("OA provider waterfall", () => {
  test("uses PMC before Unpaywall and records OA evidence", async () => {
    const calls = [];
    const result = await downloadOpenAccessArticle({ doi: "10.1234/example", title: "OA Example", pmcid: "PMC123" }, {
      email: "researcher@example.org",
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "oa-")),
      fetchImpl: async (url) => {
        calls.push(String(url));
        if (String(url).includes("oa.fcgi")) return new Response(`<OA><records><record><link format="pdf" href="https://ftp.ncbi.nlm.nih.gov/a.pdf" /></record></records></OA>`);
        if (String(url).includes("unpaywall")) return Response.json({ is_oa: true, best_oa_location: { url_for_pdf: "https://repo.example/a.pdf" }, oa_locations: [] });
        if (String(url).includes("ftp.ncbi")) return new Response(Buffer.from("%PDF-1.7\nPMC"), { headers: { "content-type": "application/pdf" } });
        throw new Error(`unexpected URL ${url}`);
      },
    });
    assert.equal(result.status, "open_access_downloaded");
    assert.equal(result.oaEvidence.source, "pmc");
    assert.equal(calls.some((url) => url.includes("repo.example")), false);
  });

  test("reports oa_not_found when no lawful candidate exists", async () => {
    const result = await downloadOpenAccessArticle({ doi: "10.1234/closed", title: "Closed" }, {
      email: "researcher@example.org",
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "oa-")),
      fetchImpl: async (url) => String(url).includes("unpaywall")
        ? Response.json({ is_oa: false, oa_locations: [] })
        : new Response("not found", { status: 404 }),
    });
    assert.equal(result.status, "oa_not_found");
    assert.equal(result.oaAssessment, "confirmed_closed");
  });

  test("keeps OA state unknown when required resolvers could not decide", async () => {
    const result = await downloadOpenAccessArticle({ doi: "10.1234/unknown", title: "Unknown" }, {
      email: "",
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "oa-")),
      fetchImpl: async () => new Response("unavailable", { status: 503 }),
    });
    assert.equal(result.status, "oa_not_found");
    assert.equal(result.oaAssessment, "unknown");
  });

  test("does not reclassify a confirmed OA article as closed when its file is temporarily unavailable", async () => {
    const result = await downloadOpenAccessArticle({ doi: "10.1234/oa", title: "OA", license: "cc-by", publisherPdfUrl: "https://publisher.example/a.pdf" }, {
      email: "researcher@example.org",
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "oa-")),
      fetchImpl: async (url) => String(url).includes("unpaywall")
        ? Response.json({ is_oa: true, best_oa_location: { url_for_pdf: "https://repo.example/a.pdf", host_type: "repository" }, oa_locations: [] })
        : String(url).includes("europepmc")
          ? Response.json({ resultList: { result: [] } })
          : new Response("temporary", { status: 503 }),
    });
    assert.equal(result.oaAssessment, "confirmed_oa");
    assert.equal(result.oaAttempts.length >= 1, true);
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import {
  arxivPdfUrl,
  exactTitleMatch,
  filenameForPdfUrl,
  normalizeArxivId,
  parseArxivAtom,
  parsePmcOaXml,
  parseUnpaywallRecord,
  rankOaCandidates,
} from "../../scripts/lib/open-access.mjs";

describe("open access helpers", () => {
  test("normalizes arXiv ids from ids and URLs", () => {
    assert.equal(normalizeArxivId("1706.03762"), "1706.03762");
    assert.equal(normalizeArxivId("1706.03762v7"), "1706.03762v7");
    assert.equal(normalizeArxivId("https://arxiv.org/abs/1706.03762"), "1706.03762");
    assert.equal(normalizeArxivId("https://arxiv.org/pdf/1706.03762.pdf"), "1706.03762");
  });

  test("builds canonical arXiv PDF URL", () => {
    assert.equal(arxivPdfUrl("1706.03762"), "https://arxiv.org/pdf/1706.03762");
  });

  test("matches exact titles with whitespace and case tolerance", () => {
    assert.equal(exactTitleMatch("Attention Is All You Need", "attention is all you need"), true);
    assert.equal(exactTitleMatch("Attention\nIs   All You Need", "Attention Is All You Need"), true);
    assert.equal(exactTitleMatch("Getting the attention you need", "Attention Is All You Need"), false);
  });

  test("parses exact arXiv Atom result", () => {
    const xml = `<?xml version="1.0"?>
      <feed>
        <entry>
          <id>http://arxiv.org/abs/1706.03762v7</id>
          <title>Attention Is All You Need</title>
        </entry>
        <entry>
          <id>http://arxiv.org/abs/0000.00000</id>
          <title>Getting the attention you need</title>
        </entry>
      </feed>`;
    assert.deepEqual(parseArxivAtom(xml, "Attention Is All You Need"), {
      id: "1706.03762v7",
      title: "Attention Is All You Need",
      pdfUrl: "https://arxiv.org/pdf/1706.03762v7",
    });
  });

  test("creates readable PDF filenames", () => {
    assert.equal(
      filenameForPdfUrl("https://arxiv.org/pdf/1706.03762", "Attention Is All You Need"),
      "Attention_Is_All_You_Need.pdf"
    );
    assert.equal(
      filenameForPdfUrl("https://example.org/papers/a/b/c.pdf"),
      "c.pdf"
    );
  });

  test("parses and ranks PMC before Unpaywall OA candidates", () => {
    const pmc = parsePmcOaXml(`<OA><records><record id="PMC123"><link format="pdf" href="ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/a.pdf" /></record></records></OA>`);
    const unpaywall = parseUnpaywallRecord({
      is_oa: true,
      best_oa_location: { url_for_pdf: "https://repo.example/a.pdf", host_type: "repository", version: "acceptedVersion", license: "cc-by" },
      oa_locations: [],
    });
    const ranked = rankOaCandidates([...unpaywall, ...pmc]);
    assert.equal(ranked[0].source, "pmc");
    assert.equal(ranked[1].source, "repository");
    assert.equal(ranked[1].version, "acceptedVersion");
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import {
  classifyFullTextContent,
  classifyProviderFailure,
} from "../../scripts/lib/provider-utils.mjs";

describe("native full-text validation", () => {
  test("distinguishes PDF, CAJ, HTML, JATS/XML, and login/error pages", () => {
    assert.equal(classifyFullTextContent({ contentType: "application/pdf", head: Buffer.from("%PDF-1.7") }).format, "pdf");
    assert.equal(classifyFullTextContent({ contentType: "application/octet-stream", head: Buffer.from("CAJViewer") }).format, "caj");
    assert.equal(classifyFullTextContent({ contentType: "text/html", head: Buffer.from("<!doctype html><article>Full text</article>") }).format, "html");
    assert.equal(classifyFullTextContent({ contentType: "application/xml", head: Buffer.from("<?xml version=\"1.0\"?><article article-type=\"research-article\">") }).format, "jats_xml");
    assert.equal(classifyFullTextContent({ contentType: "text/html", head: Buffer.from("<!doctype html><title>Log in</title><form>Password</form>") }).valid, false);
  });

  test("maps API authorization and empty-fulltext failures", () => {
    assert.equal(classifyProviderFailure({ status: 401 }), "credentials_invalid");
    assert.equal(classifyProviderFailure({ status: 403 }), "api_not_entitled");
    assert.equal(classifyProviderFailure({ status: 404 }), "api_fulltext_unavailable");
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { downloadPublisherArticle } from "../../scripts/lib/publisher-providers.mjs";

function response(body, { status = 200, contentType = "application/pdf" } = {}) {
  return new Response(body, { status, headers: { "content-type": contentType } });
}

describe("publisher API providers", () => {
  test("requires provider credentials without exposing a secret", async () => {
    const result = await downloadPublisherArticle({ doi: "10.1016/example", title: "Example" }, {
      provider: "elsevier",
      credentials: null,
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "provider-")),
      fetchImpl: async () => { throw new Error("must not fetch"); },
    });
    assert.equal(result.status, "credentials_missing");
    assert.match(result.configureUrl, /elsevier\.com/);
  });

  test("downloads and validates Elsevier PDF full text", async () => {
    const outDir = fs.mkdtempSync(path.join(os.tmpdir(), "provider-"));
    const result = await downloadPublisherArticle({ doi: "10.1016/example", title: "Example" }, {
      provider: "elsevier",
      credentials: { api_key: "key", insttoken: "token" },
      outDir,
      fetchImpl: async (_url, options) => {
        assert.equal(options.headers["X-ELS-APIKey"], "key");
        assert.equal(options.headers["X-ELS-Insttoken"], "token");
        return response(Buffer.from("%PDF-1.7\ncontent"));
      },
    });
    assert.equal(result.status, "downloaded");
    assert.equal(result.format, "pdf");
    assert.equal(fs.existsSync(result.file), true);
    assert.match(result.sha256, /^[a-f0-9]{64}$/);
  });

  test("accepts Springer JATS XML as native full text", async () => {
    const outDir = fs.mkdtempSync(path.join(os.tmpdir(), "provider-"));
    const secret = "springer-secret";
    const result = await downloadPublisherArticle({ doi: "10.1007/example", title: "Springer Example" }, {
      provider: "springer_nature",
      credentials: { api_key: secret },
      outDir,
      fetchImpl: async () => response(`<?xml version="1.0"?><article article-type="research-article"><body>Text</body></article>`, { contentType: "application/xml" }),
    });
    assert.equal(result.status, "native_fulltext_downloaded");
    assert.equal(result.format, "jats_xml");
    assert.match(result.file, /\.xml$/);
    assert.doesNotMatch(result.source, new RegExp(secret));
    assert.doesNotMatch(result.source, /api_key=/i);
  });

  test("reports readable publisher HTML with the HTML-specific success status", async () => {
    const outDir = fs.mkdtempSync(path.join(os.tmpdir(), "provider-"));
    const result = await downloadPublisherArticle({ doi: "10.1007/example", title: "HTML Example" }, {
      provider: "springer_nature",
      credentials: { api_key: "key" },
      outDir,
      fetchImpl: async () => response(`<!doctype html><article class="article-body">${"Full text ".repeat(100)}</article>`, { contentType: "text/html" }),
    });
    assert.equal(result.status, "full_text_html_available");
    assert.equal(result.format, "html");
  });

  test("reports entitlement failures before web fallback", async () => {
    const result = await downloadPublisherArticle({ doi: "10.1109/example", title: "IEEE Example" }, {
      provider: "ieee",
      credentials: { api_key: "key", fulltext_endpoint: "https://fulltext.ieee.test/articles/{doi}" },
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "provider-")),
      fetchImpl: async () => response("Forbidden", { status: 403, contentType: "text/plain" }),
    });
    assert.equal(result.status, "api_not_entitled");
    assert.equal(result.fallbackConfirmationRequired, true);
  });

  test("does not mistake an IEEE metadata key for paid full-text access", async () => {
    const result = await downloadPublisherArticle({ doi: "10.1109/example", title: "IEEE Example" }, {
      provider: "ieee",
      credentials: { api_key: "metadata-key" },
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "provider-")),
      fetchImpl: async () => { throw new Error("must not call metadata API as full text"); },
    });
    assert.equal(result.status, "api_fulltext_unavailable");
    assert.match(result.detail, /Full-Text Access API/i);
  });

  test("retries transient Elsevier failures in the canonical provider", async () => {
    let calls = 0;
    const result = await downloadPublisherArticle({ doi: "10.1016/example", title: "Retry Example" }, {
      provider: "elsevier",
      credentials: { api_key: "key" },
      outDir: fs.mkdtempSync(path.join(os.tmpdir(), "provider-")),
      fetchImpl: async () => ++calls === 1
        ? response("temporary", { status: 503, contentType: "text/plain" })
        : response(Buffer.from("%PDF-1.7\ncontent")),
      sleepImpl: async () => {},
    });
    assert.equal(calls, 2);
    assert.equal(result.status, "downloaded");
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import {
  classifyPublisher,
  chooseDirectUrlRoute,
  chooseRoute,
  hasUsablePublisherCredentials,
  parseSiChoice,
} from "../../scripts/lib/routing.mjs";

describe("SI confirmation gate", () => {
  test("requires one explicit SI choice before downloading", () => {
    assert.deepEqual(parseSiChoice({ si: false, noSi: false }), {
      confirmed: false,
      status: "si_confirmation_required",
    });
    assert.deepEqual(parseSiChoice({ si: true, noSi: false }), {
      confirmed: true,
      wantSi: true,
    });
    assert.deepEqual(parseSiChoice({ si: false, noSi: true }), {
      confirmed: true,
      wantSi: false,
    });
    assert.throws(() => parseSiChoice({ si: true, noSi: true }), /mutually exclusive/i);
  });
});

describe("literature routing", () => {
  test("routes every Chinese article only to CNKI", () => {
    assert.equal(chooseRoute({ title: "乡村振兴研究", isOa: true, publisher: "Elsevier" }).provider, "cnki");
    assert.equal(chooseRoute({ language: "zh", doi: "10.1016/example", isOa: false }).provider, "cnki");
    assert.equal(chooseRoute({ sourceUrl: "https://kns.cnki.net/kcms/detail/detail.aspx?filename=x" }).provider, "cnki");
  });

  test("does not let a non-CNKI PDF URL bypass Chinese routing", () => {
    assert.deepEqual(chooseDirectUrlRoute({
      pdfUrl: "https://example.org/paper.pdf",
      title: "乡村振兴研究",
    }), { provider: "cnki", mode: "title_search", reason: "chinese_literature" });
    assert.deepEqual(chooseDirectUrlRoute({
      pdfUrl: "https://example.org/paper.pdf",
      language: "zh-CN",
    }), { provider: "cnki", mode: "title_search", reason: "chinese_literature" });
  });

  test("keeps CNKI direct URLs on the direct CNKI path", () => {
    assert.deepEqual(chooseDirectUrlRoute({
      pdfUrl: "https://kns.cnki.net/kcms/detail/detail.aspx?filename=x",
      title: "乡村振兴研究",
    }), { provider: "cnki", mode: "direct", reason: "explicit_cnki_url" });
  });

  test("routes supported publishers with credentials to their API before OA", () => {
    const route = chooseRoute({ language: "en", isOa: true, publisher: "Elsevier", hasPublisherCredentials: true });
    assert.equal(route.provider, "elsevier");
    assert.equal(route.reason, "publisher_api_credentials_available");
  });

  test("requires an IEEE full-text endpoint before treating its credentials as usable", () => {
    assert.equal(hasUsablePublisherCredentials("elsevier", { api_key: "key" }), true);
    assert.equal(hasUsablePublisherCredentials("ieee", { api_key: "metadata-key" }), false);
    assert.equal(hasUsablePublisherCredentials("ieee", {
      api_key: "fulltext-key",
      fulltext_endpoint: "https://ieee.example/articles/{doi}",
    }), true);
  });

  test("routes an English OA article without publisher credentials to OA", () => {
    assert.equal(chooseRoute({ language: "en", isOa: true, publisher: "Elsevier", hasPublisherCredentials: false }).provider, "open_access");
  });

  test("routes English non-OA articles by publisher", () => {
    assert.equal(chooseRoute({ language: "en", isOa: false, publisher: "Elsevier" }).provider, "elsevier");
    assert.equal(chooseRoute({ language: "en", isOa: false, publisher: "Springer Nature" }).provider, "springer_nature");
    assert.equal(chooseRoute({ language: "en", isOa: false, publisher: "IEEE" }).provider, "ieee");
    assert.equal(chooseRoute({ language: "en", isOa: false, publisher: "Wiley" }).provider, "web_access");
  });

  test("recognizes publisher from DOI prefixes and names", () => {
    assert.equal(classifyPublisher({ doi: "10.1016/j.cell.2026.01.001" }), "elsevier");
    assert.equal(classifyPublisher({ doi: "10.1007/s00122-021-03957-1" }), "springer_nature");
    assert.equal(classifyPublisher({ doi: "10.1109/5.771073" }), "ieee");
    assert.equal(classifyPublisher({ publisher: "Elsevier B.V." }), "elsevier");
  });
});

import { test, describe } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
  configPathFromEnv,
  loadSchoolConfig,
  discoveryUrlFromConfig,
  DEFAULT_DISCOVERY_URL,
} from "../../scripts/lib/school-config.mjs";

describe("school config loader", () => {
  test("uses LIT_DL_CONFIG_DIR when present", () => {
    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lit-dl-"));
    assert.equal(configPathFromEnv({ LIT_DL_CONFIG_DIR: dir }), path.join(dir, "school.json"));
  });

  test("returns null for missing config", () => {
    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lit-dl-"));
    assert.equal(loadSchoolConfig({ LIT_DL_CONFIG_DIR: dir }), null);
  });

  test("returns configured discovery URL", () => {
    const config = {
      discovery: {
        web_of_science_url: "https://example.edu/wos",
      },
    };
    assert.equal(discoveryUrlFromConfig(config), "https://example.edu/wos");
  });

  test("falls back to default Web of Science URL", () => {
    assert.equal(discoveryUrlFromConfig(null), DEFAULT_DISCOVERY_URL);
    assert.equal(discoveryUrlFromConfig({}), DEFAULT_DISCOVERY_URL);
  });
});

// Unit tests for scripts/lib/status-codes.mjs and scripts/lib/pdf-utils.mjs
// Run: node --test tests/unit/status-codes.test.mjs

import { test, describe } from "node:test";
import assert from "node:assert/strict";
import {
  STATUS,
  classifyWall,
  mapLegacyStatus,
  isUserHandoff,
  isSuccess,
} from "../../scripts/lib/status-codes.mjs";
import { isHtmlResponse, isPdfHead, shouldRejectHtmlResponse } from "../../scripts/lib/pdf-utils.mjs";

describe("classifyWall", () => {
  test("generic CAS URL -> carsi_waiting_user", () => {
    const r = classifyWall("https://login.university.example/authserver/login?service=x", "Login", "");
    assert.equal(r.status, STATUS.CARSI_WAITING_USER);
    assert.match(r.reason, /institutional/);
  });

  test("generic IdP URL -> carsi_waiting_user", () => {
    const r = classifyWall("https://identity.university.example/idp/profile/SAML2/Redirect/SSO", "", "");
    assert.equal(r.status, STATUS.CARSI_WAITING_USER);
  });

  test("carsi.edu.cn URL -> carsi_waiting_user", () => {
    const r = classifyWall("https://ds.carsi.edu.cn/wayf", "", "");
    assert.equal(r.status, STATUS.CARSI_WAITING_USER);
  });

  test("shibboleth URL -> carsi_waiting_user", () => {
    const r = classifyWall("https://publisher.com/shibboleth", "", "");
    assert.equal(r.status, STATUS.CARSI_WAITING_USER);
  });

  test("ScienceDirect + 'Are you a robot?' -> sciencedirect_robot_check", () => {
    const r = classifyWall(
      "https://www.sciencedirect.com/science/article/pii/Sxxx",
      "Are you a robot?",
      "unusual traffic"
    );
    assert.equal(r.status, STATUS.SCIENCEDIRECT_ROBOT_CHECK);
  });

  test("non-ScienceDirect + 'Are you a robot?' -> publisher_verification_waiting_user", () => {
    const r = classifyWall(
      "https://www.springer.com/article/10.1007/x",
      "Are you a robot?",
      ""
    );
    assert.equal(r.status, STATUS.PUBLISHER_VERIFICATION_WAITING_USER);
  });

  test("Cloudflare challenge -> publisher_verification_waiting_user", () => {
    const r = classifyWall("https://onlinelibrary.wiley.com/doi/x", "", "cloudflare challenge");
    assert.equal(r.status, STATUS.PUBLISHER_VERIFICATION_WAITING_USER);
  });

  test("CAPTCHA -> publisher_verification_waiting_user", () => {
    const r = classifyWall("https://publisher.com/x", "Verify", "please complete captcha");
    assert.equal(r.status, STATUS.PUBLISHER_VERIFICATION_WAITING_USER);
  });

  test("Access Denied -> publisher_blocked_waiting_user", () => {
    const r = classifyWall("https://publisher.com/x", "Access Denied", "403 forbidden");
    assert.equal(r.status, STATUS.PUBLISHER_BLOCKED_WAITING_USER);
  });

  test("Forbidden -> publisher_blocked_waiting_user", () => {
    const r = classifyWall("https://publisher.com/x", "Forbidden", "");
    assert.equal(r.status, STATUS.PUBLISHER_BLOCKED_WAITING_USER);
  });

  test("normal publisher page -> null (no wall)", () => {
    const r = classifyWall(
      "https://www.nature.com/articles/s41586-021-03819-2",
      "Article title",
      "Abstract text..."
    );
    assert.equal(r, null);
  });

  test("publisher page with 'Log in' link -> null (not a wall)", () => {
    // Publisher pages legitimately contain "Log in" links and must not be
    // misclassified as needing the user.
    const r = classifyWall(
      "https://www.sciencedirect.com/science/article/pii/Sxxx",
      "Article",
      "Log in to access full text"
    );
    assert.equal(r, null);
  });

  test("empty inputs -> null", () => {
    const r = classifyWall("", "", "");
    assert.equal(r, null);
  });
});

describe("mapLegacyStatus", () => {
  test("needs_user_login -> carsi_waiting_user", () => {
    assert.equal(mapLegacyStatus("needs_user_login"), STATUS.CARSI_WAITING_USER);
  });

  test("needs_user_verify -> publisher_verification_waiting_user", () => {
    assert.equal(mapLegacyStatus("needs_user_verify"), STATUS.PUBLISHER_VERIFICATION_WAITING_USER);
  });

  test("publisher_blocked -> publisher_blocked_waiting_user", () => {
    assert.equal(mapLegacyStatus("publisher_blocked"), STATUS.PUBLISHER_BLOCKED_WAITING_USER);
  });

  test("no_pdf_link -> no_full_text_link", () => {
    assert.equal(mapLegacyStatus("no_pdf_link"), STATUS.NO_FULL_TEXT_LINK);
  });

  test("error -> failed_after_retry", () => {
    assert.equal(mapLegacyStatus("error"), STATUS.FAILED_AFTER_RETRY);
  });

  test("canonical status passes through unchanged", () => {
    assert.equal(mapLegacyStatus("downloaded"), "downloaded");
    assert.equal(mapLegacyStatus("carsi_waiting_user"), "carsi_waiting_user");
  });

  test("unknown status passes through unchanged", () => {
    assert.equal(mapLegacyStatus("some_new_status"), "some_new_status");
  });

  test("null/undefined pass through", () => {
    assert.equal(mapLegacyStatus(null), null);
    assert.equal(mapLegacyStatus(undefined), undefined);
  });
});

describe("isUserHandoff", () => {
  test("carsi_waiting_user is handoff", () => {
    assert.equal(isUserHandoff(STATUS.CARSI_WAITING_USER), true);
  });

  test("publisher_verification_waiting_user is handoff", () => {
    assert.equal(isUserHandoff(STATUS.PUBLISHER_VERIFICATION_WAITING_USER), true);
  });

  test("sciencedirect_robot_check is handoff", () => {
    assert.equal(isUserHandoff(STATUS.SCIENCEDIRECT_ROBOT_CHECK), true);
  });

  test("downloaded is NOT handoff", () => {
    assert.equal(isUserHandoff(STATUS.DOWNLOADED), false);
  });

  test("failed_after_retry is NOT handoff", () => {
    assert.equal(isUserHandoff(STATUS.FAILED_AFTER_RETRY), false);
  });
});

describe("isSuccess", () => {
  test("downloaded is success", () => {
    assert.equal(isSuccess(STATUS.DOWNLOADED), true);
  });

  test("downloaded_with_si is success", () => {
    assert.equal(isSuccess(STATUS.DOWNLOADED_WITH_SI), true);
  });

  test("open_access_downloaded is success", () => {
    assert.equal(isSuccess(STATUS.OPEN_ACCESS_DOWNLOADED), true);
  });

  test("full_text_html_available is success", () => {
    assert.equal(isSuccess(STATUS.FULL_TEXT_HTML_AVAILABLE), true);
  });

  test("carsi_waiting_user is NOT success", () => {
    assert.equal(isSuccess(STATUS.CARSI_WAITING_USER), false);
  });

  test("pdf_fetch_failed is NOT success", () => {
    assert.equal(isSuccess(STATUS.PDF_FETCH_FAILED), false);
  });

  test("library_no_permission is NOT success", () => {
    assert.equal(isSuccess(STATUS.LIBRARY_NO_PERMISSION), false);
  });

  test("native_fulltext_downloaded is success", () => {
    assert.equal(isSuccess(STATUS.NATIVE_FULLTEXT_DOWNLOADED), true);
  });
});

describe("isPdfHead", () => {
  test("%PDF- bytes -> true", () => {
    // %PDF-1.5
    const bytes = [0x25, 0x50, 0x44, 0x46, 0x2d, 0x31, 0x2e, 0x35];
    assert.equal(isPdfHead(bytes), true);
  });

  test("HTML head -> false", () => {
    // <!DOCTYPE html>
    const bytes = [0x3c, 0x21, 0x44, 0x4f, 0x43, 0x54, 0x59, 0x50];
    assert.equal(isPdfHead(bytes), false);
  });

  test("empty -> false", () => {
    assert.equal(isPdfHead([]), false);
  });

  test("null -> false", () => {
    assert.equal(isPdfHead(null), false);
  });

  test("too short -> false", () => {
    assert.equal(isPdfHead([0x25, 0x50]), false);
  });

  test("Buffer works too", () => {
    const buf = Buffer.from("%PDF-1.5");
    assert.equal(isPdfHead(buf), true);
  });
});

describe("isHtmlResponse", () => {
  test("rejects HTML login pages returned as attachments", () => {
    assert.equal(isHtmlResponse({ contentType: "text/html", head: [] }), true);
    assert.equal(isHtmlResponse({ contentType: "application/octet-stream", head: Array.from(Buffer.from("<!doctype html><title>Login")) }), true);
  });

  test("accepts expected supplement file types", () => {
    assert.equal(isHtmlResponse({ contentType: "application/pdf", head: Array.from(Buffer.from("%PDF-")) }), false);
    assert.equal(isHtmlResponse({ contentType: "application/zip", head: [80, 75, 3, 4] }), false);
  });

  test("keeps HTML rejection opt-in so unrelated download routes are unchanged", () => {
    const html = { contentType: "text/html", head: [] };
    assert.equal(shouldRejectHtmlResponse(html), false);
    assert.equal(shouldRejectHtmlResponse(html, true), true);
  });
});

import { describe, test } from "node:test";
import assert from "node:assert/strict";
import path from "node:path";
import {
  articleBundleDirectory,
  downloadWosSupportingInformation,
  exactArticleTitleMatch,
  fetchAttachmentWithNavigationFallback,
  selectSupportingInformationLinks,
  shouldUseCleanWosBundle,
  wosSearchQuery,
} from "../../scripts/lib/wos-supporting-information.mjs";
import { parseArgs } from "../../scripts/batch_download.mjs";

describe("WoS supporting information selection", () => {
  test("keeps the clean bundle behavior isolated to WoS plus --si", () => {
    assert.equal(shouldUseCleanWosBundle({ topic: "Exact title", title: "Exact title", si: true }), true);
    assert.equal(shouldUseCleanWosBundle({ topic: "rice", si: true }), false);
    assert.equal(shouldUseCleanWosBundle({ topic: "rice", si: false }), false);
    assert.equal(shouldUseCleanWosBundle({ dois: ["10.1/x"], si: true }), false);
    assert.equal(shouldUseCleanWosBundle({ pdfUrl: "https://example.org/a.pdf", si: true }), false);
    assert.equal(shouldUseCleanWosBundle({ title: "OA title", openAccess: true, si: true }), false);
  });

  test("applies one explicit SI choice to a WoS topic batch", () => {
    const args = parseArgs(["node", "batch_download.mjs", "--topic", "rice", "--si"]);
    assert.equal(args.topic, "rice");
    assert.equal(args.si, true);
  });

  test("uses a quoted exact-phrase query when a title is supplied", () => {
    assert.equal(wosSearchQuery("fallback topic", "Exact article title"), '"Exact article title"');
    assert.equal(wosSearchQuery("rice drought", ""), "rice drought");
  });

  test("selects explicit attachments and one-level supplementary pages", () => {
    const links = [
      { text: "Supplementary Information", href: "/articles/paper/supplementary-information" },
      { text: "Download supplementary table", href: "/files/table-s1.xlsx", download: "table-s1.xlsx" },
      { text: "Source Data", href: "/files/source-data.zip" },
      { text: "Supplementary Information", dataDownloadUrl: "/download/123" },
      { text: "GitHub", href: "https://github.com/example/project" },
      { text: "References", href: "/references" },
    ];

    assert.deepEqual(selectSupportingInformationLinks(links, "https://publisher.test/article"), {
      attachments: [
        { url: "https://publisher.test/files/table-s1.xlsx", label: "Download supplementary table", filename: "table-s1.xlsx" },
        { url: "https://publisher.test/files/source-data.zip", label: "Source Data", filename: "source-data.zip" },
        { url: "https://publisher.test/download/123", label: "Supplementary Information", filename: "123" },
      ],
      pages: [
        { url: "https://publisher.test/articles/paper/supplementary-information", label: "Supplementary Information" },
      ],
    });
  });

  test("recognizes Elsevier, MDPI, and Wiley attachment patterns", () => {
    const links = [
      { text: "mmc1", href: "https://publisher.test/article#mmc1", rawHref: "#mmc1" },
      { text: "mmc1", href: "/action/downloadSupplement?doi=10.1/x&file=mmc1.mp4" },
      { text: "Supplementary Materials", href: "/article_deploy/html/images/supplementary/sensors-22-02521-s001.pdf" },
      { text: "Data S1", href: "/action/downloadSupplement?doi=10.1111/x&file=pce14065-sup-0001.pdf" },
    ];
    assert.equal(selectSupportingInformationLinks(links, "https://publisher.test/article").attachments.length, 3);
  });

  test("retries a CORS-blocked attachment after navigating to its origin", async () => {
    const calls = [];
    const result = await fetchAttachmentWithNavigationFallback(
      "proxy",
      "tab",
      { url: "https://cdn.publisher.test/mmc1.pdf" },
      () => "/tmp/mmc1.pdf",
      {
        fetchImpl: async () => {
          calls.push("fetch");
          return calls.length === 1 ? { ok: false, err: "TypeError: Failed to fetch" } : { ok: true, file: "/tmp/mmc1.pdf" };
        },
        navigateImpl: async () => { calls.push("navigate"); },
        waitForCompleteImpl: async () => { calls.push("wait"); },
      }
    );
    assert.equal(result.ok, true);
    assert.deepEqual(calls, ["fetch", "navigate", "wait", "fetch"]);
  });
});

describe("clean article bundle", () => {
  test("uses the exact readable title as the article folder", () => {
    assert.equal(
      articleBundleDirectory("/tmp/out", "Leaf direction: Lamina joint development and environmental responses"),
      path.join("/tmp/out", "Leaf direction Lamina joint development and environmental responses")
    );
  });

  test("matches requested and discovered titles exactly with punctuation tolerance", () => {
    assert.equal(
      exactArticleTitleMatch(
        "Sounds emitted by plants under stress are airborne and informative",
        "Sounds emitted by plants under stress are airborne and informative."
      ),
      true
    );
    assert.equal(exactArticleTitleMatch("Leaf direction", "Leaf direction: Lamina joint development and environmental responses"), false);
  });

  test("downloads direct attachments plus attachments found one page deep", async () => {
    const bundleDir = "/tmp/out/Article";
    const scanned = [];
    const result = await downloadWosSupportingInformation({
      proxy: "proxy",
      tab: "tab",
      landingUrl: "https://publisher.test/article",
      bundleDir,
      dependencies: {
        scanPageImpl: async (_proxy, _tab, url) => {
          scanned.push(url);
          if (url.endsWith("/article")) {
            return [
              { text: "Supplementary Information", href: "/supplement-page" },
              { text: "Source Data", href: "/files/source-data.zip" },
            ];
          }
          return [{ text: "Supplementary Table", href: "/files/table-s1.xlsx" }];
        },
        fetchAttachmentImpl: async (_proxy, _tab, attachment, resolvePath) => {
          const file = resolvePath({
            finalUrl: attachment.url,
            contentType: attachment.url.endsWith("zip") ? "application/zip" : "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
            contentDisposition: `attachment; filename=${attachment.filename}`,
          });
          return { ok: true, file, bytes: 100 };
        },
      },
    });

    assert.deepEqual(scanned, [
      "https://publisher.test/article",
      "https://publisher.test/supplement-page",
    ]);
    assert.equal(result.status, "downloaded");
    assert.equal(result.downloaded, 2);
    assert.deepEqual(result.files.sort(), [
      path.join(bundleDir, "source-data.zip"),
      path.join(bundleDir, "table-s1.xlsx"),
    ]);
  });

  test("reports not_found without creating diagnostic files", async () => {
    const result = await downloadWosSupportingInformation({
      proxy: "proxy",
      tab: "tab",
      landingUrl: "https://publisher.test/article",
      bundleDir: "/tmp/out/Article",
      dependencies: { scanPageImpl: async () => [] },
    });
    assert.deepEqual(result, {
      status: "not_found",
      found: 0,
      downloaded: 0,
      files: [],
      failures: [],
    });
  });

  test("adds a MIME extension and never overwrites the main PDF", async () => {
    const bundleDir = "/tmp/out/Article";
    const result = await downloadWosSupportingInformation({
      proxy: "proxy",
      tab: "tab",
      landingUrl: "https://publisher.test/article",
      bundleDir,
      reservedFilenames: ["Article.pdf"],
      dependencies: {
        scanPageImpl: async () => [{ text: "Supplementary", href: "/downloadSupplement?file=Article.pdf" }],
        fetchAttachmentImpl: async (_proxy, _tab, attachment, resolvePath) => ({
          ok: true,
          file: resolvePath({
            url: "https://publisher.test/downloadSupplement",
            contentType: "application/pdf",
            contentDisposition: "attachment; filename=Article.pdf",
          }),
          bytes: 100,
        }),
      },
    });
    assert.deepEqual(result.files, [path.join(bundleDir, "Article-2.pdf")]);

    const extensionless = await downloadWosSupportingInformation({
      proxy: "proxy",
      tab: "tab",
      landingUrl: "https://publisher.test/article",
      bundleDir,
      dependencies: {
        scanPageImpl: async () => [{ text: "Supplementary", href: "/downloadSupplement?file=supplement" }],
        fetchAttachmentImpl: async (_proxy, _tab, attachment, resolvePath) => ({
          ok: true,
          file: resolvePath({ url: attachment.url, contentType: "application/pdf", contentDisposition: "" }),
          bytes: 100,
        }),
      },
    });
    assert.deepEqual(extensionless.files, [path.join(bundleDir, "supplement.pdf")]);
  });

  test("reports an inaccessible supplementary page as fetch_failed", async () => {
    const result = await downloadWosSupportingInformation({
      proxy: "proxy",
      tab: "tab",
      landingUrl: "https://publisher.test/article",
      bundleDir: "/tmp/out/Article",
      dependencies: {
        scanPageImpl: async (_proxy, _tab, url) => {
          if (url.endsWith("/article")) return [{ text: "Supplementary Information", href: "/supplement-page" }];
          throw new Error("publisher verification required");
        },
      },
    });
    assert.equal(result.status, "fetch_failed");
    assert.match(result.failures[0].error, /verification required/);
  });
});