arXiv API from JavaScript

// Markup: <ol id="papers"></ol>
const ATOM = 'http://www.w3.org/2005/Atom';
const ARXIV = 'http://arxiv.org/schemas/atom';
const OPENSEARCH = 'http://a9.com/-/spec/opensearch/1.1/';

async function searchArxiv(query, { start = 0, max = 20, sortBy = 'relevance' } = {}) {
  const params = new URLSearchParams({
    search_query: query,
    start,
    max_results: max,
    sortBy,
    sortOrder: 'descending',
  });
  const response = await fetch(`https://proxy.cors.dev/https://export.arxiv.org/api/query?${params}`, {
    credentials: 'omit',
  });
  if (!response.ok) {
    throw new Error(`arXiv request failed: ${response.headers.get('X-Cors-Error') ?? response.status}`);
  }

  const feed = new DOMParser().parseFromString(await response.text(), 'application/xml');
  if (feed.querySelector('parsererror')) throw new Error('The response is not valid XML');

  const text = (node, name) => node.getElementsByTagNameNS(ATOM, name)[0]?.textContent.trim() ?? '';
  const papers = [...feed.getElementsByTagNameNS(ATOM, 'entry')].map((entry) => ({
    id: text(entry, 'id').split('/abs/')[1],
    title: text(entry, 'title').replace(/\s+/g, ' '),
    authors: [...entry.getElementsByTagNameNS(ATOM, 'author')].map((author) => text(author, 'name')),
    summary: text(entry, 'summary').replace(/\s+/g, ' '),
    published: text(entry, 'published').slice(0, 10),
    category: entry.getElementsByTagNameNS(ARXIV, 'primary_category')[0]?.getAttribute('term'),
  }));

  return { total: Number(feed.getElementsByTagNameNS(OPENSEARCH, 'totalResults')[0]?.textContent), papers };
}

const list = document.querySelector('#papers');
searchArxiv('cat:cs.AI AND ti:agent', { max: 10, sortBy: 'submittedDate' })
  .then(({ total, papers }) => {
    list.replaceChildren(
      ...papers.map((paper) => {
        const item = document.createElement('li');
        const link = document.createElement('a');
        link.href = `https://arxiv.org/abs/${paper.id}`;
        link.textContent = paper.title;
        item.append(link, ` (${paper.published}, ${paper.authors.slice(0, 3).join(', ')})`);
        return item;
      }),
    );
    console.log(`${total} papers match`);
  })
  .catch((error) => {
    list.textContent = `Could not load papers: ${error.message}`;
  });

// Uses searchArxiv() from the example above, in a module script or the DevTools console
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

async function collect(query, { pageSize = 200, limit = 1000 } = {}) {
  const papers = [];
  for (let start = 0; start < limit; start += pageSize) {
    const page = await searchArxiv(query, { start, max: pageSize, sortBy: 'submittedDate' });
    papers.push(...page.papers);
    if (page.papers.length < pageSize || start + pageSize >= page.total) break;
    await sleep(3000);
  }
  return papers.slice(0, limit);
}

const recent = await collect('cat:cs.CL', { pageSize: 200, limit: 600 });
console.log(recent.length, recent[0]);