From cc8c1c32d1ad3f9846e5ee834f9fbf23b361424f Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 02:28:10 +0800 Subject: [PATCH 1/9] add searchWithMeta() with totalResults and updated timestamp in return value --- src/__snapshots__/index.test.js.snap | 35 ++++++++++++++++++++++++++++ src/index.js | 22 ++++++++++++++++- src/index.test.js | 19 ++++++++++++++- 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/src/__snapshots__/index.test.js.snap b/src/__snapshots__/index.test.js.snap index 7e490a7..a857707 100644 --- a/src/__snapshots__/index.test.js.snap +++ b/src/__snapshots__/index.test.js.snap @@ -93,6 +93,41 @@ Array [ ] `; +exports[`arXiv search tests should return results with meta data as expected 1`] = ` +Object { + "entries": Array [ + Object { + "authors": Array [ + Array [ + "AUTHOR1", + ], + Array [ + "AUTHOR2", + ], + ], + "categories": Array [ + "CATEGORY", + ], + "id": "PAPER_ID", + "links": Array [ + Object { + "href": "URL", + "rel": "REL", + "title": "TITLE", + "type": "TYPE", + }, + ], + "published": "PUBLISHED", + "summary": "SUMMARY", + "title": "TITLE", + "updated": "UPDATED", + }, + ], + "totalResults": "1", + "updated": "2021-05-16T00:00:00-04:00", +} +`; + exports[`arXiv search tests should throw error - exclude tags is not an array 1`] = `[Error: include and exclude must be arrays]`; exports[`arXiv search tests should throw error - include tags empty 1`] = `[Error: include is a mandatory field]`; diff --git a/src/index.js b/src/index.js index 77e6555..4fdce55 100644 --- a/src/index.js +++ b/src/index.js @@ -74,6 +74,22 @@ function parseTags({include, exclude = []}) { * @returns {Promise} */ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) { + const result = await searchWithMeta({searchQueryParams, sortBy, sortOrder, start, maxResults}); + return result.entries; +} + +/** + * Fetch data from arXiv API + * @async + * @param {{searchQueryParams: Array.<{include: Array, exclude: Array}>, start: number, maxResults: number}} args + * @param {Array} searchQueryParams - array of search query. + * @param {string} sortBy - can be "relevance", "lastUpdatedDate", "submittedDate". + * @param {string} sortOrder - can be either "ascending" or "descending". + * @param {number} start - the index of the first returned result. + * @param {number} maxResults - the number of results returned by the query. + * @returns {Promise<{entries, totalResults, updated}>} + */ +async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) { if (!Array.isArray(searchQueryParams)) { throw new Error('query param must be an array'); } @@ -86,9 +102,13 @@ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResul const searchQuery = searchQueryParams.map(parseTags).join(SEPARATORS.OR); const response = await axios.get(get_arxiv_url({searchQuery, sortBy, sortOrder, start, maxResults})); const parsedData = await parseStringPromisified(response.data); - return _.get(parsedData, 'feed.entry', []).map(parseArxivObject); + const entries = _.get(parsedData, 'feed.entry', []).map(parseArxivObject); + const totalResults = _.get(parsedData, 'feed.opensearch:totalResults[0]_'); + const updated = _.get(parsedData, 'feed.updated[0]'); + return {entries, totalResults, updated}; } module.exports = { search, + searchWithMeta, }; diff --git a/src/index.test.js b/src/index.test.js index 8d04b31..c33bbbe 100644 --- a/src/index.test.js +++ b/src/index.test.js @@ -2,6 +2,8 @@ const {PREFIXES, SORT_BY, SORT_ORDER} = require('./constants'); const mockResponse = { feed: { + updated: ['2021-05-16T00:00:00-04:00'], + 'opensearch:totalResults': [{_: '1'}], entry: [ { id: ['PAPER_ID'], @@ -28,7 +30,7 @@ jest.mock('util', () => ({ promisify: jest.fn(() => mockXmlPromisify), })); -const {search} = require('./index.js'); +const {search, searchWithMeta} = require('./index.js'); describe('arXiv search tests', () => { beforeEach(() => { @@ -91,6 +93,21 @@ describe('arXiv search tests', () => { ); expect(results).toMatchSnapshot(); }); + it('should return results with meta data as expected', async () => { + const results = await searchWithMeta({ + searchQueryParams: [ + { + include: [{name: 'GAN'}], + }, + ], + start: 10, + maxResults: 50, + }); + expect(mockAxiosGet).toHaveBeenCalledWith( + 'http://export.arxiv.org/api/query?search_query=all:GAN&start=10&max_results=50' + ); + expect(results).toMatchSnapshot(); + }); it('should throw error - unsupported sortBy', async () => { await expect( search({ From dc035cc6c2df9276d54e5e5b2f476ff299936de5 Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 02:33:24 +0800 Subject: [PATCH 2/9] update contributors list --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index 9fdfc9e..f101795 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,9 @@ "javascript" ], "author": "elior avraham", + "contributors": [ + "Beeno Tung (https://beeno-tung.surge.sh)" + ], "license": "ISC", "bugs": { "url": "https://github.com/eliorav/arXiv-api/issues" From 1c31d0d259b8020b8b9761194dc4e2085e9806e2 Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 02:57:51 +0800 Subject: [PATCH 3/9] export searchWithMeta in top-level module --- index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index f032a2b..51ea8e0 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,6 @@ -const {search} = require('./src'); +const {search, searchWithMeta} = require('./src'); module.exports = { search, + searchWithMeta, }; From e2a22773a545a2be21bb62d5dd9105554f30dfca Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 03:58:19 +0800 Subject: [PATCH 4/9] extract and export parseResponseData() for reuse example use cases: - parse data from file - parse data downloadeded from other library --- index.js | 7 +------ src/index.js | 14 +++++++++++++- src/index.test.js | 24 +++++++++++++++++++++++- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 51ea8e0..3fc7798 100644 --- a/index.js +++ b/index.js @@ -1,6 +1 @@ -const {search, searchWithMeta} = require('./src'); - -module.exports = { - search, - searchWithMeta, -}; +module.exports = require('./src'); diff --git a/src/index.js b/src/index.js index 4fdce55..d12183d 100644 --- a/src/index.js +++ b/src/index.js @@ -101,7 +101,18 @@ async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, } const searchQuery = searchQueryParams.map(parseTags).join(SEPARATORS.OR); const response = await axios.get(get_arxiv_url({searchQuery, sortBy, sortOrder, start, maxResults})); - const parsedData = await parseStringPromisified(response.data); + const result = await module.exports.parseResponseData(response.data); + return result; +} + +/** + * Parse data from arXiv API + * @async + * @param {{toString(): string}} convertableToString - can be string or Buffer + * @returns {Promise<{entries, totalResults, updated}>} + */ +async function parseResponseData(convertableToString) { + const parsedData = await parseStringPromisified(convertableToString); const entries = _.get(parsedData, 'feed.entry', []).map(parseArxivObject); const totalResults = _.get(parsedData, 'feed.opensearch:totalResults[0]_'); const updated = _.get(parsedData, 'feed.updated[0]'); @@ -111,4 +122,5 @@ async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, module.exports = { search, searchWithMeta, + parseResponseData, }; diff --git a/src/index.test.js b/src/index.test.js index c33bbbe..c3c5c40 100644 --- a/src/index.test.js +++ b/src/index.test.js @@ -30,7 +30,7 @@ jest.mock('util', () => ({ promisify: jest.fn(() => mockXmlPromisify), })); -const {search, searchWithMeta} = require('./index.js'); +const {search, searchWithMeta, parseResponseData} = require('./index.js'); describe('arXiv search tests', () => { beforeEach(() => { @@ -108,6 +108,28 @@ describe('arXiv search tests', () => { ); expect(results).toMatchSnapshot(); }); + it('should export response parser', () => { + expect(parseResponseData).toBeDefined(); + expect(typeof parseResponseData).toBe('function'); + }); + it('should use response parser', async () => { + const src = require('./index'); + const original = src.parseResponseData; + const spy = jest.fn(original); + src.parseResponseData = spy; + try { + await src.search({ + searchQueryParams: [ + { + include: [{name: 'GAN'}], + }, + ], + }); + expect(spy).toHaveBeenCalledWith('XML'); + } finally { + src.parseResponseData = original; + } + }); it('should throw error - unsupported sortBy', async () => { await expect( search({ From 01e807d3ef442fe403858a5c53b9ec27a8aadc99 Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 04:05:58 +0800 Subject: [PATCH 5/9] parse totalResults into number --- src/__snapshots__/index.test.js.snap | 2 +- src/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/__snapshots__/index.test.js.snap b/src/__snapshots__/index.test.js.snap index a857707..b3b3565 100644 --- a/src/__snapshots__/index.test.js.snap +++ b/src/__snapshots__/index.test.js.snap @@ -123,7 +123,7 @@ Object { "updated": "UPDATED", }, ], - "totalResults": "1", + "totalResults": 1, "updated": "2021-05-16T00:00:00-04:00", } `; diff --git a/src/index.js b/src/index.js index d12183d..a5c5790 100644 --- a/src/index.js +++ b/src/index.js @@ -114,7 +114,7 @@ async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, async function parseResponseData(convertableToString) { const parsedData = await parseStringPromisified(convertableToString); const entries = _.get(parsedData, 'feed.entry', []).map(parseArxivObject); - const totalResults = _.get(parsedData, 'feed.opensearch:totalResults[0]_'); + const totalResults = +_.get(parsedData, 'feed.opensearch:totalResults[0]_'); const updated = _.get(parsedData, 'feed.updated[0]'); return {entries, totalResults, updated}; } From 9bd609644d347a13c2fa99a6be528a3564790b69 Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 04:43:03 +0800 Subject: [PATCH 6/9] include link, startIndex, itemsPerPage in search result meta data --- src/__snapshots__/index.test.js.snap | 5 ++++- src/index.js | 12 +++++++++++- src/index.test.js | 5 ++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/__snapshots__/index.test.js.snap b/src/__snapshots__/index.test.js.snap index b3b3565..e83a4ae 100644 --- a/src/__snapshots__/index.test.js.snap +++ b/src/__snapshots__/index.test.js.snap @@ -123,7 +123,10 @@ Object { "updated": "UPDATED", }, ], - "totalResults": 1, + "itemsPerPage": 1, + "link": "LINK", + "startIndex": 0, + "totalResults": 2, "updated": "2021-05-16T00:00:00-04:00", } `; diff --git a/src/index.js b/src/index.js index a5c5790..7ba92a2 100644 --- a/src/index.js +++ b/src/index.js @@ -114,9 +114,19 @@ async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, async function parseResponseData(convertableToString) { const parsedData = await parseStringPromisified(convertableToString); const entries = _.get(parsedData, 'feed.entry', []).map(parseArxivObject); + const link = decodeURIComponent(_.get(parsedData, 'feed.link[0].$.href')); const totalResults = +_.get(parsedData, 'feed.opensearch:totalResults[0]_'); + const startIndex = +_.get(parsedData, 'feed.opensearch:startIndex[0]_'); + const itemsPerPage = +_.get(parsedData, 'feed.opensearch:itemsPerPage[0]_'); const updated = _.get(parsedData, 'feed.updated[0]'); - return {entries, totalResults, updated}; + return { + link, + totalResults, + startIndex, + itemsPerPage, + updated, + entries, + }; } module.exports = { diff --git a/src/index.test.js b/src/index.test.js index c3c5c40..bcffa71 100644 --- a/src/index.test.js +++ b/src/index.test.js @@ -2,8 +2,11 @@ const {PREFIXES, SORT_BY, SORT_ORDER} = require('./constants'); const mockResponse = { feed: { + link: [{$: {href: 'LINK'}}], updated: ['2021-05-16T00:00:00-04:00'], - 'opensearch:totalResults': [{_: '1'}], + 'opensearch:totalResults': [{_: '2'}], + 'opensearch:startIndex': [{_: '0'}], + 'opensearch:itemsPerPage': [{_: '1'}], entry: [ { id: ['PAPER_ID'], From 51cfdbce638a014f853f10d0cde1bff714f5ffa3 Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 04:43:15 +0800 Subject: [PATCH 7/9] test against sample xml file --- package.json | 4 +-- test/__snapshots__/sample.test.js.snap | 48 ++++++++++++++++++++++++++ test/sample.test.js | 11 ++++++ test/sample.xml | 26 ++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 test/__snapshots__/sample.test.js.snap create mode 100644 test/sample.test.js create mode 100644 test/sample.xml diff --git a/package.json b/package.json index f101795..a4943b5 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "node wrapper for arXiv api", "main": "index.js", "scripts": { - "test": "jest -u ./src/index.test.js", - "test:coverage": "jest ./src/index.test.js --coverage", + "test": "jest -u ./src/index.test.js ./test/sample.test.js", + "test:coverage": "jest ./src/index.test.js ./test/sample.test.js --coverage", "lint": "eslint '**/*.js' --quiet" }, "repository": { diff --git a/test/__snapshots__/sample.test.js.snap b/test/__snapshots__/sample.test.js.snap new file mode 100644 index 0000000..9ac1200 --- /dev/null +++ b/test/__snapshots__/sample.test.js.snap @@ -0,0 +1,48 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`test response data parser with sample xml file should not throw errors 1`] = ` +Object { + "entries": Array [ + Object { + "authors": Array [ + Array [ + "sample author name", + ], + ], + "categories": Array [ + Object { + "scheme": "http://arxiv.org/schemas/atom", + "term": "cs.CV", + }, + Object { + "scheme": "http://arxiv.org/schemas/atom", + "term": "cs.LG", + }, + ], + "id": "http://arxiv.org/abs/sample_code", + "links": Array [ + Object { + "href": "http://arxiv.org/abs/sample_code", + "rel": "alternate", + "type": "text/html", + }, + Object { + "href": "http://arxiv.org/pdf/sample_code", + "rel": "related", + "title": "pdf", + "type": "application/pdf", + }, + ], + "published": "2019-04-01T12:19:28Z", + "summary": "sample summary", + "title": "sample title", + "updated": "2019-04-01T12:19:28Z", + }, + ], + "itemsPerPage": 1, + "link": "http://arxiv.org/api/query?search_query=all:GAN&id_list=&start=0&max_results=1", + "startIndex": 0, + "totalResults": 6123, + "updated": "2021-05-16T00:00:00-04:00", +} +`; diff --git a/test/sample.test.js b/test/sample.test.js new file mode 100644 index 0000000..78e7552 --- /dev/null +++ b/test/sample.test.js @@ -0,0 +1,11 @@ +const {readFileSync} = require('fs'); +const {join} = require('path'); +const {parseResponseData} = require('../src/index'); + +describe('test response data parser with sample xml file', () => { + it('should not throw errors', async () => { + const text = readFileSync(join(__dirname, 'sample.xml')); + const result = await parseResponseData(text); + await expect(result).toMatchSnapshot(); + }); +}); diff --git a/test/sample.xml b/test/sample.xml new file mode 100644 index 0000000..12d3d4f --- /dev/null +++ b/test/sample.xml @@ -0,0 +1,26 @@ + + + + ArXiv Query: search_query=all:GAN&id_list=&start=0&max_results=1 + http://arxiv.org/api/sample_code + 2021-05-16T00:00:00-04:00 + 6123 + 0 + 1 + + http://arxiv.org/abs/sample_code + 2019-04-01T12:19:28Z + 2019-04-01T12:19:28Z + sample title + sample summary + + sample author name + + 3 pages + + + + + + + \ No newline at end of file From b555adaa4bf6944c27cd8e31ab555c1f33d1d32f Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 05:04:14 +0800 Subject: [PATCH 8/9] update returns type in comment --- src/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/index.js b/src/index.js index 7ba92a2..fa43e05 100644 --- a/src/index.js +++ b/src/index.js @@ -71,7 +71,7 @@ function parseTags({include, exclude = []}) { * @param {string} sortOrder - can be either "ascending" or "descending". * @param {number} start - the index of the first returned result. * @param {number} maxResults - the number of results returned by the query. - * @returns {Promise} + * @returns {Promise} */ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) { const result = await searchWithMeta({searchQueryParams, sortBy, sortOrder, start, maxResults}); @@ -87,7 +87,7 @@ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResul * @param {string} sortOrder - can be either "ascending" or "descending". * @param {number} start - the index of the first returned result. * @param {number} maxResults - the number of results returned by the query. - * @returns {Promise<{entries, totalResults, updated}>} + * @returns {Promise<{link, totalResults, startIndex, itemsPerPage, updated, entries}>} */ async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) { if (!Array.isArray(searchQueryParams)) { @@ -109,7 +109,7 @@ async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, * Parse data from arXiv API * @async * @param {{toString(): string}} convertableToString - can be string or Buffer - * @returns {Promise<{entries, totalResults, updated}>} + * @returns {Promise<{link, totalResults, startIndex, itemsPerPage, updated, entries}>} */ async function parseResponseData(convertableToString) { const parsedData = await parseStringPromisified(convertableToString); From c90c1a2319d17947dd27b9833eae366f0e4b0539 Mon Sep 17 00:00:00 2001 From: Beeno Tung Date: Mon, 17 May 2021 05:21:04 +0800 Subject: [PATCH 9/9] add typescript types --- index.d.ts | 63 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 11 ++++++-- test/typescript.ts | 16 ++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 index.d.ts create mode 100644 test/typescript.ts diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..86ef819 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,63 @@ +import { convertableToString } from 'xml2js' + +export type SearchResult = { + link: string // query {search_query,start,max_results,sortBy,sortOrder} + updated: string // iso timestamp + totalResults: number + startIndex: number + itemsPerPage: number + entries: Entry[] +} +export type Entry = { + id: string // url + title: string + summary: string + authors: string[][] + links: Link[] + published: string // iso timestamp + updated: string // iso timestamp + categories: Category[] +} +export type Link = { + title?: 'pdf' + href: string + ref: 'alternate' | 'related' + type: 'text/html' | 'application/pdf' +} +export type Category = { + term: string + scheme: string +} + +export type Options = { + searchQueryParams: SearchQueryParam[] + sortBy?: 'relevance' | 'lastUpdatedDate' | 'submittedDate' + sortOrder?: 'ascending' | 'descending' + start?: number // default 0 + maxResults?: number // default 10 +} +export type SearchQueryParam = { + include: SearchTerm[] // non-empty + exclude?: SearchTerm[] +} +export type SearchTerm = { + name: string + prefix?: Prefix // default 'all' +} +export type Prefix = + | 'all' + | 'ti' // Title + | 'au' // Author + | 'abs' // Abstract + | 'co' // Comment + | 'jr' // Journal Reference + | 'cat' // Subject Category + | 'rn' // Report Number + +export function search(options: Options): Promise + +export function searchWithMeta(options: Options): Promise + +export function parseResponseData( + convertableToString: convertableToString, +): Promise diff --git a/package.json b/package.json index a4943b5..c5ec544 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,10 @@ "description": "node wrapper for arXiv api", "main": "index.js", "scripts": { - "test": "jest -u ./src/index.test.js ./test/sample.test.js", + "test": "run-p test:index test:sample test:typescript", + "test:index": "jest -u ./src/index.test.js", + "test:sample": "jest -u ./test/sample.test.js", + "test:typescript": "ts-node test/typescript.ts", "test:coverage": "jest ./src/index.test.js ./test/sample.test.js --coverage", "lint": "eslint '**/*.js' --quiet" }, @@ -32,13 +35,17 @@ "xml2js": "0.4.23" }, "devDependencies": { + "@types/xml2js": "^0.4.8", "eslint": "6.8.0", "eslint-config-prettier": "6.10.1", "eslint-plugin-jest": "23.8.2", "eslint-plugin-node": "11.0.0", "eslint-plugin-prettier": "^2.6.0", "jest": "25.2.2", - "prettier": "1.16.4" + "npm-run-all": "^4.1.5", + "prettier": "1.16.4", + "ts-node": "^9.1.1", + "typescript": "^4.2.4" }, "engines": { "node": ">=8.10.0" diff --git a/test/typescript.ts b/test/typescript.ts new file mode 100644 index 0000000..fd04ea3 --- /dev/null +++ b/test/typescript.ts @@ -0,0 +1,16 @@ +import { readFileSync } from 'fs' +import { join } from 'path' +import { parseResponseData } from '../index' + +let data = readFileSync(join(__dirname, 'sample.xml')) +parseResponseData(data) + .then(res => { + if (res.totalResults !== 6123) { + throw new Error('invalid result') + } + console.log('typescript setup is working') + }) + .catch(err => { + console.error(err) + process.exit(1) + })