diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..86ef819 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,63 @@ +import { convertableToString } from 'xml2js' + +export type SearchResult = { + link: string // query {search_query,start,max_results,sortBy,sortOrder} + updated: string // iso timestamp + totalResults: number + startIndex: number + itemsPerPage: number + entries: Entry[] +} +export type Entry = { + id: string // url + title: string + summary: string + authors: string[][] + links: Link[] + published: string // iso timestamp + updated: string // iso timestamp + categories: Category[] +} +export type Link = { + title?: 'pdf' + href: string + ref: 'alternate' | 'related' + type: 'text/html' | 'application/pdf' +} +export type Category = { + term: string + scheme: string +} + +export type Options = { + searchQueryParams: SearchQueryParam[] + sortBy?: 'relevance' | 'lastUpdatedDate' | 'submittedDate' + sortOrder?: 'ascending' | 'descending' + start?: number // default 0 + maxResults?: number // default 10 +} +export type SearchQueryParam = { + include: SearchTerm[] // non-empty + exclude?: SearchTerm[] +} +export type SearchTerm = { + name: string + prefix?: Prefix // default 'all' +} +export type Prefix = + | 'all' + | 'ti' // Title + | 'au' // Author + | 'abs' // Abstract + | 'co' // Comment + | 'jr' // Journal Reference + | 'cat' // Subject Category + | 'rn' // Report Number + +export function search(options: Options): Promise + +export function searchWithMeta(options: Options): Promise + +export function parseResponseData( + convertableToString: convertableToString, +): Promise diff --git a/index.js b/index.js index f032a2b..3fc7798 100644 --- a/index.js +++ b/index.js @@ -1,5 +1 @@ -const {search} = require('./src'); - -module.exports = { - search, -}; +module.exports = require('./src'); diff --git a/package.json b/package.json index 9fdfc9e..c5ec544 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,11 @@ "description": "node wrapper for arXiv api", "main": "index.js", "scripts": { - "test": "jest -u ./src/index.test.js", - "test:coverage": "jest ./src/index.test.js --coverage", + "test": "run-p test:index test:sample test:typescript", + "test:index": "jest -u ./src/index.test.js", + "test:sample": "jest -u ./test/sample.test.js", + "test:typescript": "ts-node test/typescript.ts", + "test:coverage": "jest ./src/index.test.js ./test/sample.test.js --coverage", "lint": "eslint '**/*.js' --quiet" }, "repository": { @@ -18,6 +21,9 @@ "javascript" ], "author": "elior avraham", + "contributors": [ + "Beeno Tung (https://beeno-tung.surge.sh)" + ], "license": "ISC", "bugs": { "url": "https://github.com/eliorav/arXiv-api/issues" @@ -29,13 +35,17 @@ "xml2js": "0.4.23" }, "devDependencies": { + "@types/xml2js": "^0.4.8", "eslint": "6.8.0", "eslint-config-prettier": "6.10.1", "eslint-plugin-jest": "23.8.2", "eslint-plugin-node": "11.0.0", "eslint-plugin-prettier": "^2.6.0", "jest": "25.2.2", - "prettier": "1.16.4" + "npm-run-all": "^4.1.5", + "prettier": "1.16.4", + "ts-node": "^9.1.1", + "typescript": "^4.2.4" }, "engines": { "node": ">=8.10.0" diff --git a/src/__snapshots__/index.test.js.snap b/src/__snapshots__/index.test.js.snap index 7e490a7..e83a4ae 100644 --- a/src/__snapshots__/index.test.js.snap +++ b/src/__snapshots__/index.test.js.snap @@ -93,6 +93,44 @@ Array [ ] `; +exports[`arXiv search tests should return results with meta data as expected 1`] = ` +Object { + "entries": Array [ + Object { + "authors": Array [ + Array [ + "AUTHOR1", + ], + Array [ + "AUTHOR2", + ], + ], + "categories": Array [ + "CATEGORY", + ], + "id": "PAPER_ID", + "links": Array [ + Object { + "href": "URL", + "rel": "REL", + "title": "TITLE", + "type": "TYPE", + }, + ], + "published": "PUBLISHED", + "summary": "SUMMARY", + "title": "TITLE", + "updated": "UPDATED", + }, + ], + "itemsPerPage": 1, + "link": "LINK", + "startIndex": 0, + "totalResults": 2, + "updated": "2021-05-16T00:00:00-04:00", +} +`; + exports[`arXiv search tests should throw error - exclude tags is not an array 1`] = `[Error: include and exclude must be arrays]`; exports[`arXiv search tests should throw error - include tags empty 1`] = `[Error: include is a mandatory field]`; diff --git a/src/index.js b/src/index.js index 77e6555..fa43e05 100644 --- a/src/index.js +++ b/src/index.js @@ -71,9 +71,25 @@ function parseTags({include, exclude = []}) { * @param {string} sortOrder - can be either "ascending" or "descending". * @param {number} start - the index of the first returned result. * @param {number} maxResults - the number of results returned by the query. - * @returns {Promise} + * @returns {Promise} */ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) { + const result = await searchWithMeta({searchQueryParams, sortBy, sortOrder, start, maxResults}); + return result.entries; +} + +/** + * Fetch data from arXiv API + * @async + * @param {{searchQueryParams: Array.<{include: Array, exclude: Array}>, start: number, maxResults: number}} args + * @param {Array} searchQueryParams - array of search query. + * @param {string} sortBy - can be "relevance", "lastUpdatedDate", "submittedDate". + * @param {string} sortOrder - can be either "ascending" or "descending". + * @param {number} start - the index of the first returned result. + * @param {number} maxResults - the number of results returned by the query. + * @returns {Promise<{link, totalResults, startIndex, itemsPerPage, updated, entries}>} + */ +async function searchWithMeta({searchQueryParams, sortBy, sortOrder, start = 0, maxResults = 10}) { if (!Array.isArray(searchQueryParams)) { throw new Error('query param must be an array'); } @@ -85,10 +101,36 @@ async function search({searchQueryParams, sortBy, sortOrder, start = 0, maxResul } const searchQuery = searchQueryParams.map(parseTags).join(SEPARATORS.OR); const response = await axios.get(get_arxiv_url({searchQuery, sortBy, sortOrder, start, maxResults})); - const parsedData = await parseStringPromisified(response.data); - return _.get(parsedData, 'feed.entry', []).map(parseArxivObject); + const result = await module.exports.parseResponseData(response.data); + return result; +} + +/** + * Parse data from arXiv API + * @async + * @param {{toString(): string}} convertableToString - can be string or Buffer + * @returns {Promise<{link, totalResults, startIndex, itemsPerPage, updated, entries}>} + */ +async function parseResponseData(convertableToString) { + const parsedData = await parseStringPromisified(convertableToString); + const entries = _.get(parsedData, 'feed.entry', []).map(parseArxivObject); + const link = decodeURIComponent(_.get(parsedData, 'feed.link[0].$.href')); + const totalResults = +_.get(parsedData, 'feed.opensearch:totalResults[0]_'); + const startIndex = +_.get(parsedData, 'feed.opensearch:startIndex[0]_'); + const itemsPerPage = +_.get(parsedData, 'feed.opensearch:itemsPerPage[0]_'); + const updated = _.get(parsedData, 'feed.updated[0]'); + return { + link, + totalResults, + startIndex, + itemsPerPage, + updated, + entries, + }; } module.exports = { search, + searchWithMeta, + parseResponseData, }; diff --git a/src/index.test.js b/src/index.test.js index 8d04b31..bcffa71 100644 --- a/src/index.test.js +++ b/src/index.test.js @@ -2,6 +2,11 @@ const {PREFIXES, SORT_BY, SORT_ORDER} = require('./constants'); const mockResponse = { feed: { + link: [{$: {href: 'LINK'}}], + updated: ['2021-05-16T00:00:00-04:00'], + 'opensearch:totalResults': [{_: '2'}], + 'opensearch:startIndex': [{_: '0'}], + 'opensearch:itemsPerPage': [{_: '1'}], entry: [ { id: ['PAPER_ID'], @@ -28,7 +33,7 @@ jest.mock('util', () => ({ promisify: jest.fn(() => mockXmlPromisify), })); -const {search} = require('./index.js'); +const {search, searchWithMeta, parseResponseData} = require('./index.js'); describe('arXiv search tests', () => { beforeEach(() => { @@ -91,6 +96,43 @@ describe('arXiv search tests', () => { ); expect(results).toMatchSnapshot(); }); + it('should return results with meta data as expected', async () => { + const results = await searchWithMeta({ + searchQueryParams: [ + { + include: [{name: 'GAN'}], + }, + ], + start: 10, + maxResults: 50, + }); + expect(mockAxiosGet).toHaveBeenCalledWith( + 'http://export.arxiv.org/api/query?search_query=all:GAN&start=10&max_results=50' + ); + expect(results).toMatchSnapshot(); + }); + it('should export response parser', () => { + expect(parseResponseData).toBeDefined(); + expect(typeof parseResponseData).toBe('function'); + }); + it('should use response parser', async () => { + const src = require('./index'); + const original = src.parseResponseData; + const spy = jest.fn(original); + src.parseResponseData = spy; + try { + await src.search({ + searchQueryParams: [ + { + include: [{name: 'GAN'}], + }, + ], + }); + expect(spy).toHaveBeenCalledWith('XML'); + } finally { + src.parseResponseData = original; + } + }); it('should throw error - unsupported sortBy', async () => { await expect( search({ diff --git a/test/__snapshots__/sample.test.js.snap b/test/__snapshots__/sample.test.js.snap new file mode 100644 index 0000000..9ac1200 --- /dev/null +++ b/test/__snapshots__/sample.test.js.snap @@ -0,0 +1,48 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`test response data parser with sample xml file should not throw errors 1`] = ` +Object { + "entries": Array [ + Object { + "authors": Array [ + Array [ + "sample author name", + ], + ], + "categories": Array [ + Object { + "scheme": "http://arxiv.org/schemas/atom", + "term": "cs.CV", + }, + Object { + "scheme": "http://arxiv.org/schemas/atom", + "term": "cs.LG", + }, + ], + "id": "http://arxiv.org/abs/sample_code", + "links": Array [ + Object { + "href": "http://arxiv.org/abs/sample_code", + "rel": "alternate", + "type": "text/html", + }, + Object { + "href": "http://arxiv.org/pdf/sample_code", + "rel": "related", + "title": "pdf", + "type": "application/pdf", + }, + ], + "published": "2019-04-01T12:19:28Z", + "summary": "sample summary", + "title": "sample title", + "updated": "2019-04-01T12:19:28Z", + }, + ], + "itemsPerPage": 1, + "link": "http://arxiv.org/api/query?search_query=all:GAN&id_list=&start=0&max_results=1", + "startIndex": 0, + "totalResults": 6123, + "updated": "2021-05-16T00:00:00-04:00", +} +`; diff --git a/test/sample.test.js b/test/sample.test.js new file mode 100644 index 0000000..78e7552 --- /dev/null +++ b/test/sample.test.js @@ -0,0 +1,11 @@ +const {readFileSync} = require('fs'); +const {join} = require('path'); +const {parseResponseData} = require('../src/index'); + +describe('test response data parser with sample xml file', () => { + it('should not throw errors', async () => { + const text = readFileSync(join(__dirname, 'sample.xml')); + const result = await parseResponseData(text); + await expect(result).toMatchSnapshot(); + }); +}); diff --git a/test/sample.xml b/test/sample.xml new file mode 100644 index 0000000..12d3d4f --- /dev/null +++ b/test/sample.xml @@ -0,0 +1,26 @@ + + + + ArXiv Query: search_query=all:GAN&id_list=&start=0&max_results=1 + http://arxiv.org/api/sample_code + 2021-05-16T00:00:00-04:00 + 6123 + 0 + 1 + + http://arxiv.org/abs/sample_code + 2019-04-01T12:19:28Z + 2019-04-01T12:19:28Z + sample title + sample summary + + sample author name + + 3 pages + + + + + + + \ No newline at end of file diff --git a/test/typescript.ts b/test/typescript.ts new file mode 100644 index 0000000..fd04ea3 --- /dev/null +++ b/test/typescript.ts @@ -0,0 +1,16 @@ +import { readFileSync } from 'fs' +import { join } from 'path' +import { parseResponseData } from '../index' + +let data = readFileSync(join(__dirname, 'sample.xml')) +parseResponseData(data) + .then(res => { + if (res.totalResults !== 6123) { + throw new Error('invalid result') + } + console.log('typescript setup is working') + }) + .catch(err => { + console.error(err) + process.exit(1) + })