Merge pull request #16 from g0vhk-io/develop

Promote develop to master
g0vhk-io · May 6, 2019 · 694234a · 694234a
2 parents 42ad928 + f456cd9
commit 694234a
Show file tree

Hide file tree

Showing 46 changed files with 26,386 additions and 794 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,22 @@
+## 0.3.0 (2019-05-06)
+
+* feature: add batch search (#15) ([573fc9d](https://github.com/g0vhk-io/hk-address-parser-lib/commit/573fc9d)), closes [#15](https://github.com/g0vhk-io/hk-address-parser-lib/issues/15)
+* add documentation pages for the project (#13) ([e723296](https://github.com/g0vhk-io/hk-address-parser-lib/commit/e723296)), closes [#13](https://github.com/g0vhk-io/hk-address-parser-lib/issues/13)
+* Bugfix/fix return type of latlng (#12) ([d1e360f](https://github.com/g0vhk-io/hk-address-parser-lib/commit/d1e360f)), closes [#12](https://github.com/g0vhk-io/hk-address-parser-lib/issues/12)
+* enhance accuracy test (#14) ([f8802ae](https://github.com/g0vhk-io/hk-address-parser-lib/commit/f8802ae)), closes [#14](https://github.com/g0vhk-io/hk-address-parser-lib/issues/14)
+
+
+
+## 0.3.0 (2019-05-06)
+
+* feature: add batch search (#15) ([573fc9d](https://github.com/g0vhk-io/hk-address-parser-lib/commit/573fc9d)), closes [#15](https://github.com/g0vhk-io/hk-address-parser-lib/issues/15)
+* add documentation pages for the project (#13) ([e723296](https://github.com/g0vhk-io/hk-address-parser-lib/commit/e723296)), closes [#13](https://github.com/g0vhk-io/hk-address-parser-lib/issues/13)
+* Bugfix/fix return type of latlng (#12) ([d1e360f](https://github.com/g0vhk-io/hk-address-parser-lib/commit/d1e360f)), closes [#12](https://github.com/g0vhk-io/hk-address-parser-lib/issues/12)
+* enhance accuracy test (#14) ([f8802ae](https://github.com/g0vhk-io/hk-address-parser-lib/commit/f8802ae)), closes [#14](https://github.com/g0vhk-io/hk-address-parser-lib/issues/14)
+
+
+
+# [0.3.0](https://github.com/g0vhk-io/hk-address-parser-lib/compare/0.2.4...0.3.0) (2019-05-06)
+
+
+
diff --git a/README.md b/README.md
@@ -1,17 +1,17 @@
 # 香港地址解析器 Hong Kong Address Parser Lib
 
-This is the JavaScript Address Resolver library for [Hong Kong Address Parser](https://g0vhk-io.github.io/HKAddressParser).  
-Feel free to check the [frontend repo](https://github.com/g0vhk-io/HKAddressParser) or download the [npm package](https://www.npmjs.com/package/hk-address-parser-lib).  
+This is the JavaScript Address Resolver library for [Hong Kong Address Parser](https://g0vhk-io.github.io/HKAddressParser).
+Feel free to check the [frontend repo](https://github.com/g0vhk-io/HKAddressParser) or download the [npm package](https://www.npmjs.com/package/hk-address-parser-lib).
+
+
+For full details, see [https://g0vhk-io.github.io/hk-address-parser-lib](https://g0vhk-io.github.io/hk-address-parser-lib)
 
 ## Installation
 
-You are required to install these two dependencies yourself if they are not already in your project:
-```bash
-npm i @turf/turf proj4
-```
-Then install the address parser:
+Install library from npm directly
+
 ```bash
-npm i hk-address-parser
+npm i hk-address-parser-lib
 ```
 
 ## Usage
@@ -27,30 +27,3 @@ records.forEach(address => {
   //
 })
 ```
-
----
-
-### API for address
-
-#### function components(lang: LANG): any
-
-#### function componentLabelForKey(key: string, lang: LANG): string
-
-#### function componentValueForKey(key: string, lang: LANG): string
-
-#### function fullAddress(lang: LANG): string
-
-#### function coordinate(): { lat, lng }
-
-#### function coordinates(): { lat, lng }[]
-
-#### function dataSource(): string[]
-
-#### function confidence(): int
-
-#### function distanceTo(address: Address): Number
-
-## Contribute
-
-### Testing the accuracy
-
diff --git a/accuracy_test/full_test.js b/accuracy_test/full_test.js
@@ -47,7 +47,7 @@ function toRad(Value) {
 
 /**
  * Load the test cases from file
- * @param {string} filePath 
+ * @param {string} filePath
  */
 async function readTestCases(filePath) {
   return new Promise((resolve, reject) => {
@@ -141,81 +141,133 @@ function loadFromCache(url) {
 
 // Replace the fetch function
 const nodeFetch = global.fetch;
-global.fetch = function (...args) {
-  const url = args[0];
+const metrics = {
+  loadedFromCache: 0,
+  totalRequest: 0,
+  repeatedRequest: 0,
+  uriHash: {},
+}
+
+const resourceHash = {};
 
-  // try to load the cached files
-  const cache = loadFromCache(url);
-  if (cache) {
+function replaceGlobalFetchWithMemCached() {
+  global.fetch = function (...args) {
+    const url = args[0];
     return new Promise((resolve) => {
       resolve({
-        json: () => JSON.parse(cache)
+        json: () => resourceHash[url]
       });
     });
-  } else {
-    // Hook the function
-    const fn = new Promise((resolve, reject) => {
-      nodeFetch(...args).then(res => {
-        // Overwrite the json()
-        res.json().then(json => {
-          saveToCache(url, json);
-          res.json = () => json;
-          resolve(res);
-        })
-
-      })
-    });
-    return fn;
   }
 }
 
-async function main({ limit = Infinity, outputFile }) {
-  return new Promise(async (resolve, reject) => {
-    const startTime = moment();
-    const allTestData = await readTestCases(__dirname + '/test_cases/testcases_ogcio_searchable.csv');
-    const result ={ 
-      total: 0
+function replaceGlobalFetchWithFileCache() {
+  global.fetch = function (...args) {
+    const url = args[0];
+
+    metrics.totalRequest += 1;
+    if (metrics.uriHash[url] === undefined) {
+      metrics.uriHash[url] = 1;
+    } else {
+      metrics.uriHash[url] += 1;
     }
 
+    // try to load the cached files
+    const cache = loadFromCache(url);
+    if (cache) {
+      metrics.loadedFromCache += 1;
+      resourceHash[url] = JSON.parse(cache);
+      return new Promise((resolve) => {
+        resolve({
+          json: () => JSON.parse(cache)
+        });
+      });
+    } else {
+      // Hook the function
+      const fn = new Promise((resolve, reject) => {
+        nodeFetch(...args).then(res => {
+          // Overwrite the json()
+          res.json().then(json => {
+            saveToCache(url, json);
+            resourceHash[url] = json;
+            res.json = () => json;
+            resolve(res);
+          })
 
-    result.date = moment().format('YYYY-MM-DD hh:mm:ss');
-    if (typeof(tag) === 'string' && tag.length > 0) {
-      result.tag = tag;
+        })
+      });
+      return fn;
     }
-
-    result.success = 0;
-    result.failed = [];
+  }
+}
+
+
 
+async function main({ limit = Infinity, outputFile, verbose = false }) {
+  return new Promise(async (resolve, reject) => {
+    // In this test we do not want to count the io/network time
+    // So we should dry run the first test data and preload all the required network/file io responses.
+
+    const allTestData = await readTestCases(__dirname + '/test_cases/testcases_ogcio_searchable.csv');
+    // Replace the fetch with file/network
+    replaceGlobalFetchWithFileCache();
+
+    // dry run to load all the reponse to memory first
     async.eachOfLimit(allTestData.slice(0, limit), 2000, async (testData) => {
-      result.total += 1;
-      try {
-        const [address, lat, lng] = testData;
-        const jsResult = await runTest(address);      
-        if (checkResult(jsResult, lat, lng)) {
-          result.success += 1;
-        } else {
-          result.failed.push(address);
-        }
-      } catch (error) {
-        error(`Error when running ${testData}`);
-        error(error);
-      }
+      const [address] = testData;
+      await runTest(address);
     }, // callback
-      () => {
-        log(`Finished! Total ${result.total} tests executed .`);
+      async () => {
+        // Real run the test
+        replaceGlobalFetchWithMemCached();
+
+        const startTime = moment();
+        const result ={
+          total: 0
+        }
+        result.date = moment().format('YYYY-MM-DD hh:mm:ss');
+
+
+        result.success = 0;
+        result.failed = [];
+
+        // No concurrent requests allowed
+        // Finish the test cases one by one (all responses should be in memory)
+        for (const testcase of allTestData) {
+          result.total += 1;
+          try {
+            const [address, lat, lng] = testcase;
+            const jsResult = await runTest(address);
+            if (checkResult(jsResult, lat, lng)) {
+              result.success += 1;
+            } else {
+              result.failed.push(address);
+            }
+          } catch (error) {
+            error(`Error when running ${testcase}`);
+            error(error);
+          }
+        }
+
+
         const timeElapsed = moment().diff(startTime, 'ms');
+        log(`Finished! Total ${result.total} tests executed .`);
         log(`Time elapsed: ${timeElapsed}ms`);
+        log(`========================================`);
+        log(`Total Request fired: ${metrics.totalRequest}`);
+        log(`Request cached: ${metrics.loadedFromCache}`);
+        log(`Repeated request: ${metrics.repeatedRequest}`);
+        log(`Average request per query: ${Math.round(metrics.totalRequest * 100/result.total) / 100}`);
         // Write to file
-        
+
         result.success_rate = `${result.success / result.total}`;
 
         if (outputFile) {
           outputResultTofile(result, outputFile);
-        } else {
+        }
+        if (verbose) {
           log(result);
         }
-
-
         resolve();
       })
   });
@@ -244,19 +296,18 @@ program
 program
   .description('Run the test cases')
   .option('-o, --output [file]', 'Output the test result to the file, default output to console')
-  .option('-l, --limit [n]', 'Limit the number of test cases to run')  
+  .option('-l, --limit [n]', 'Limit the number of test cases to run')
+  .option('-v, --verbose', 'Show verbose log including the failed cases')
   .parse(process.argv);
 
 
 const outputFile = program.output;
-const tag = program.tag;
-
+const verbose = program.verbose;
 // bitwise flag: | python | node |
 const limit = program.limit || Infinity;
 
-main({ limit, outputFile })
+main({ limit, outputFile, verbose })
   .then((end) => {
-    log('Done');
   })
   .catch((err) => {
     error(err);

diff --git a/babel.config.js b/babel.config.js
@@ -1,9 +1,9 @@
 module.exports = {
   env: {
     development: {
-      presets: [
-        '@vue/app'
-      ],
+      // presets: [
+      //   '@vue/app'
+      // ],
       // To allow commonjs module.exports and require syntax work probably
       // Seems vue-cli 3.0.3 onwards there will be error if not specify this
       // https://github.com/vuejs/vue-cli/issues/2675