add metadata along with search result with typescript types

index.d.ts

@@ -0,0 +1,63 @@
+import { convertableToString } from 'xml2js'
+
+export type SearchResult = {
+  link: string // query {search_query,start,max_results,sortBy,sortOrder}
+  updated: string // iso timestamp
+  totalResults: number
+  startIndex: number
+  itemsPerPage: number
+  entries: Entry[]
+}
+export type Entry = {
+  id: string // url
+  title: string
+  summary: string
+  authors: string[][]
+  links: Link[]
+  published: string // iso timestamp
+  updated: string // iso timestamp
+  categories: Category[]
+}
+export type Link = {
+  title?: 'pdf'
+  href: string
+  ref: 'alternate' | 'related'
+  type: 'text/html' | 'application/pdf'
+}
+export type Category = {
+  term: string
+  scheme: string
+}
+
+export type Options = {
+  searchQueryParams: SearchQueryParam[]
+  sortBy?: 'relevance' | 'lastUpdatedDate' | 'submittedDate'
+  sortOrder?: 'ascending' | 'descending'
+  start?: number // default 0
+  maxResults?: number // default 10
+}
+export type SearchQueryParam = {
+  include: SearchTerm[] // non-empty
+  exclude?: SearchTerm[]
+}
+export type SearchTerm = {
+  name: string
+  prefix?: Prefix // default 'all'
+}
+export type Prefix =
+  | 'all'
+  | 'ti' // Title
+  | 'au' // Author
+  | 'abs' // Abstract
+  | 'co' // Comment
+  | 'jr' // Journal Reference
+  | 'cat' // Subject Category
+  | 'rn' // Report Number
+
+export function search(options: Options): Promise<Entry[]>
+
+export function searchWithMeta(options: Options): Promise<SearchResult>
+
+export function parseResponseData(
+  convertableToString: convertableToString,
+): Promise<SearchResult>

index.js

@@ -1,5 +1 @@
-const {search} = require('./src');
-
-module.exports = {
-	search,
-};
+module.exports = require('./src');

package.json

@@ -4,8 +4,11 @@
   "description": "node wrapper for arXiv api",
   "main": "index.js",
   "scripts": {
-    "test": "jest -u ./src/index.test.js",
-    "test:coverage": "jest ./src/index.test.js --coverage",
+    "test": "run-p test:index test:sample test:typescript",
+    "test:index": "jest -u ./src/index.test.js",
+    "test:sample": "jest -u ./test/sample.test.js",
+    "test:typescript": "ts-node test/typescript.ts",
+    "test:coverage": "jest ./src/index.test.js ./test/sample.test.js --coverage",
     "lint": "eslint '**/*.js' --quiet"
   },
   "repository": {
@@ -18,6 +21,9 @@
     "javascript"
   ],
   "author": "elior avraham",
+  "contributors": [
+    "Beeno Tung <aabbcc1241@yahoo.com.hk> (https://beeno-tung.surge.sh)"
+  ],
   "license": "ISC",
   "bugs": {
     "url": "https://github.com/eliorav/arXiv-api/issues"
@@ -29,13 +35,17 @@
     "xml2js": "0.4.23"
   },
   "devDependencies": {
+    "@types/xml2js": "^0.4.8",
     "eslint": "6.8.0",
     "eslint-config-prettier": "6.10.1",
     "eslint-plugin-jest": "23.8.2",
     "eslint-plugin-node": "11.0.0",
     "eslint-plugin-prettier": "^2.6.0",
     "jest": "25.2.2",
-    "prettier": "1.16.4"
+    "npm-run-all": "^4.1.5",
+    "prettier": "1.16.4",
+    "ts-node": "^9.1.1",
+    "typescript": "^4.2.4"
   },
   "engines": {
     "node": ">=8.10.0"

src/__snapshots__/index.test.js.snap

@@ -93,6 +93,44 @@ Array [
 ]
 `;
 
+exports[`arXiv search tests should return results with meta data as expected 1`] = `
+Object {
+  "entries": Array [
+    Object {
+      "authors": Array [
+        Array [
+          "AUTHOR1",
+        ],
+        Array [
+          "AUTHOR2",
+        ],
+      ],
+      "categories": Array [
+        "CATEGORY",
+      ],
+      "id": "PAPER_ID",
+      "links": Array [
+        Object {
+          "href": "URL",
+          "rel": "REL",
+          "title": "TITLE",
+          "type": "TYPE",
+        },
+      ],
+      "published": "PUBLISHED",
+      "summary": "SUMMARY",
+      "title": "TITLE",
+      "updated": "UPDATED",
+    },
+  ],
+  "itemsPerPage": 1,
+  "link": "LINK",
+  "startIndex": 0,
+  "totalResults": 2,
+  "updated": "2021-05-16T00:00:00-04:00",
+}
+`;
+
 exports[`arXiv search tests should throw error - exclude tags is not an array 1`] = `[Error: include and exclude must be arrays]`;
 
 exports[`arXiv search tests should throw error - include tags empty 1`] = `[Error: include is a mandatory field]`;

src/index.js

@@ -71,9 +71,25 @@ function parseTags({include, exclude = []}) {
  * @param {string} sortOrder - can be either "ascending" or "descending".
  * @param {number} start - the index of the first returned result.
  * @param {number} maxResults - the number of results returned by the query.
- * @returns {Promise}
+ * @returns {Promise<Array>}
  */
 async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) {
+	const result = await searchWithMeta({searchQueryParams, sortBy, sortOrder, start, maxResults});
+	return result.entries;
+}
+
+/**
+ * Fetch data from arXiv API
+ * @async
+ * @param {{searchQueryParams: Array.<{include: Array, exclude: Array}>, start: number, maxResults: number}} args
+ * @param {Array} searchQueryParams - array of search query.
+ * @param {string} sortBy - can be "relevance", "lastUpdatedDate", "submittedDate".
+ * @param {string} sortOrder - can be either "ascending" or "descending".
+ * @param {number} start - the index of the first returned result.
+ * @param {number} maxResults - the number of results returned by the query.
+ * @returns {Promise<{link, totalResults, startIndex, itemsPerPage, updated, entries}>}
+ */
+async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) {
 	if (!Array.isArray(searchQueryParams)) {
 		throw new Error('query param must be an array');
 	}
@@ -85,10 +101,36 @@ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResul
 	}
 	const searchQuery = searchQueryParams.map(parseTags).join(SEPARATORS.OR);
 	const response = await axios.get(get_arxiv_url({searchQuery, sortBy, sortOrder, start, maxResults}));
-	const parsedData = await parseStringPromisified(response.data);
-	return _.get(parsedData, 'feed.entry', []).map(parseArxivObject);
+	const result = await module.exports.parseResponseData(response.data);
+	return result;
+}
+
+/**
+ * Parse data from arXiv API
+ * @async
+ * @param {{toString(): string}} convertableToString - can be string or Buffer
+ * @returns {Promise<{link, totalResults, startIndex, itemsPerPage, updated, entries}>}
+ */
+async function parseResponseData(convertableToString) {
+	const parsedData = await parseStringPromisified(convertableToString);
+	const entries = _.get(parsedData, 'feed.entry', []).map(parseArxivObject);
+	const link = decodeURIComponent(_.get(parsedData, 'feed.link[0].$.href'));
+	const totalResults = +_.get(parsedData, 'feed.opensearch:totalResults[0]_');
+	const startIndex = +_.get(parsedData, 'feed.opensearch:startIndex[0]_');
+	const itemsPerPage = +_.get(parsedData, 'feed.opensearch:itemsPerPage[0]_');
+	const updated = _.get(parsedData, 'feed.updated[0]');
+	return {
+		link,
+		totalResults,
+		startIndex,
+		itemsPerPage,
+		updated,
+		entries,
+	};
 }
 
 module.exports = {
 	search,
+	searchWithMeta,
+	parseResponseData,
 };

src/index.test.js

@@ -2,6 +2,11 @@ const {PREFIXES, SORT_BY, SORT_ORDER} = require('./constants');
 
 const mockResponse = {
 	feed: {
+		link: [{$: {href: 'LINK'}}],
+		updated: ['2021-05-16T00:00:00-04:00'],
+		'opensearch:totalResults': [{_: '2'}],
+		'opensearch:startIndex': [{_: '0'}],
+		'opensearch:itemsPerPage': [{_: '1'}],
 		entry: [
 			{
 				id: ['PAPER_ID'],
@@ -28,7 +33,7 @@ jest.mock('util', () => ({
 	promisify: jest.fn(() => mockXmlPromisify),
 }));
 
-const {search} = require('./index.js');
+const {search, searchWithMeta, parseResponseData} = require('./index.js');
 
 describe('arXiv search tests', () => {
 	beforeEach(() => {
@@ -91,6 +96,43 @@ describe('arXiv search tests', () => {
 		);
 		expect(results).toMatchSnapshot();
 	});
+	it('should return results with meta data as expected', async () => {
+		const results = await searchWithMeta({
+			searchQueryParams: [
+				{
+					include: [{name: 'GAN'}],
+				},
+			],
+			start: 10,
+			maxResults: 50,
+		});
+		expect(mockAxiosGet).toHaveBeenCalledWith(
+			'http://export.arxiv.org/api/query?search_query=all:GAN&start=10&max_results=50'
+		);
+		expect(results).toMatchSnapshot();
+	});
+	it('should export response parser', () => {
+		expect(parseResponseData).toBeDefined();
+		expect(typeof parseResponseData).toBe('function');
+	});
+	it('should use response parser', async () => {
+		const src = require('./index');
+		const original = src.parseResponseData;
+		const spy = jest.fn(original);
+		src.parseResponseData = spy;
+		try {
+			await src.search({
+				searchQueryParams: [
+					{
+						include: [{name: 'GAN'}],
+					},
+				],
+			});
+			expect(spy).toHaveBeenCalledWith('XML');
+		} finally {
+			src.parseResponseData = original;
+		}
+	});
 	it('should throw error - unsupported sortBy', async () => {
 		await expect(
 			search({

test/__snapshots__/sample.test.js.snap

@@ -0,0 +1,48 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`test response data parser with sample xml file should not throw errors 1`] = `
+Object {
+  "entries": Array [
+    Object {
+      "authors": Array [
+        Array [
+          "sample author name",
+        ],
+      ],
+      "categories": Array [
+        Object {
+          "scheme": "http://arxiv.org/schemas/atom",
+          "term": "cs.CV",
+        },
+        Object {
+          "scheme": "http://arxiv.org/schemas/atom",
+          "term": "cs.LG",
+        },
+      ],
+      "id": "http://arxiv.org/abs/sample_code",
+      "links": Array [
+        Object {
+          "href": "http://arxiv.org/abs/sample_code",
+          "rel": "alternate",
+          "type": "text/html",
+        },
+        Object {
+          "href": "http://arxiv.org/pdf/sample_code",
+          "rel": "related",
+          "title": "pdf",
+          "type": "application/pdf",
+        },
+      ],
+      "published": "2019-04-01T12:19:28Z",
+      "summary": "sample summary",
+      "title": "sample title",
+      "updated": "2019-04-01T12:19:28Z",
+    },
+  ],
+  "itemsPerPage": 1,
+  "link": "http://arxiv.org/api/query?search_query=all:GAN&id_list=&start=0&max_results=1",
+  "startIndex": 0,
+  "totalResults": 6123,
+  "updated": "2021-05-16T00:00:00-04:00",
+}
+`;

test/sample.test.js

@@ -0,0 +1,11 @@
+const {readFileSync} = require('fs');
+const {join} = require('path');
+const {parseResponseData} = require('../src/index');
+
+describe('test response data parser with sample xml file', () => {
+	it('should not throw errors', async () => {
+		const text = readFileSync(join(__dirname, 'sample.xml'));
+		const result = await parseResponseData(text);
+		await expect(result).toMatchSnapshot();
+	});
+});

test/sample.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <link href="http://arxiv.org/api/query?search_query%3Dall%3AGAN%26id_list%3D%26start%3D0%26max_results%3D1" rel="self" type="application/atom+xml"/>
+  <title type="html">ArXiv Query: search_query=all:GAN&amp;id_list=&amp;start=0&amp;max_results=1</title>
+  <id>http://arxiv.org/api/sample_code</id>
+  <updated>2021-05-16T00:00:00-04:00</updated>
+  <opensearch:totalResults xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">6123</opensearch:totalResults>
+  <opensearch:startIndex xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">0</opensearch:startIndex>
+  <opensearch:itemsPerPage xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">1</opensearch:itemsPerPage>
+  <entry>
+    <id>http://arxiv.org/abs/sample_code</id>
+    <updated>2019-04-01T12:19:28Z</updated>
+    <published>2019-04-01T12:19:28Z</published>
+    <title>sample title</title>
+    <summary>sample summary</summary>
+    <author>
+      <name>sample author name</name>
+    </author>
+    <arxiv:comment xmlns:arxiv="http://arxiv.org/schemas/atom">3 pages</arxiv:comment>
+    <link href="http://arxiv.org/abs/sample_code" rel="alternate" type="text/html"/>
+    <link title="pdf" href="http://arxiv.org/pdf/sample_code" rel="related" type="application/pdf"/>
+    <arxiv:primary_category xmlns:arxiv="http://arxiv.org/schemas/atom" term="cs.CV" scheme="http://arxiv.org/schemas/atom"/>
+    <category term="cs.CV" scheme="http://arxiv.org/schemas/atom"/>
+    <category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
+  </entry>
+</feed>

test/typescript.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from 'fs'
+import { join } from 'path'
+import { parseResponseData } from '../index'
+
+let data = readFileSync(join(__dirname, 'sample.xml'))
+parseResponseData(data)
+  .then(res => {
+    if (res.totalResults !== 6123) {
+      throw new Error('invalid result')
+    }
+    console.log('typescript setup is working')
+  })
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })