From 633b66f88c744687945da5d7aa87c91e823f3135 Mon Sep 17 00:00:00 2001 From: kanasimi Date: Sun, 4 Feb 2024 17:45:06 +0800 Subject: [PATCH] + batch_size --- Wikiapi.js | 108 ++++++++++++++++++++++++++++------------- docs/Wikiapi.html | 16 +++---- docs/global.html | 101 +++++++++++++++++++++++++++++++++++++- docs/index.html | 4 +- docs/wikiapi.js.html | 112 ++++++++++++++++++++++++++++++------------- 5 files changed, 263 insertions(+), 78 deletions(-) diff --git a/Wikiapi.js b/Wikiapi.js index 4b8c716..e3c2b02 100644 --- a/Wikiapi.js +++ b/Wikiapi.js @@ -1129,7 +1129,7 @@ assert('Mexico' in page_list[Wikiapi.KEY_subcategories], 'list category tree: [[ * @example Get all sub-categories of [[Category:Echinodermata]] with depth=2. // const wiki = new Wikiapi('commons'); -const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flated_subcategories: true })).flated_subcategories; +const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flat_subcategories: true })).flat_subcategories; // * * @memberof Wikiapi.prototype @@ -1728,13 +1728,38 @@ function Wikiapi_run_SQL(SQL, for_each_row/* , options */) { function Wikiapi_setup_layout_elements(options) { function Wikiapi_setup_layout_elements_executor(resolve, reject) { - // const wiki = this[KEY_wiki_session]; - wiki_API.setup_layout_elements(resolve, this.append_session_to_options(options)); + const wiki = this[KEY_wiki_session]; + wiki.setup_layout_elements(resolve, this.append_session_to_options(options)); } return new Promise(Wikiapi_setup_layout_elements_executor.bind(this)); } +/** + +layout_element = CeL.wiki.parse('{{Authority control}}'); +await wiki_session.setup_layout_element_to_insert(layout_element); + +page_data = await wiki.page(page_data); +parsed = page_data.parse(); +parsed.insert_layout_element(layout_element); +parsed.toString(); + +*/ +function Wikiapi_setup_layout_element_to_insert(layout_element, options) { + function Wikiapi_setup_layout_element_to_insert_executor(resolve, reject) { + const wiki = this[KEY_wiki_session]; + wiki.setup_layout_element_to_insert(layout_element, (location, error) => { + if (error) + reject(error); + else + resolve(location); + }, options); + } + + return new Promise(Wikiapi_setup_layout_element_to_insert_executor.bind(this)); +} + // -------------------------------------------------------- /** @@ -1956,6 +1981,7 @@ wiki.listen(function for_each_row() { run_SQL: Wikiapi_run_SQL, setup_layout_elements: Wikiapi_setup_layout_elements, + setup_layout_element_to_insert: Wikiapi_setup_layout_element_to_insert, }); // wrapper for properties @@ -1987,8 +2013,18 @@ console.log('All done.'); // let count = 0; for await (const page_data of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC') })) { - if (++count > 5) break; console.trace('page_data:', page_data); + if (++count > 5) break; +} +console.log('Done.'); +// + * + * @example Process all pages. +// +let count = 0; +for await (const page_list of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC'), batch_size: 5 })) { + console.trace('page_list:', page_list); + if (++count > 2) break; } console.log('Done.'); // @@ -2022,6 +2058,27 @@ for (const type of wiki_API.list.type_list) { //console.trace(arguments); let done, page_queue = [], resolve_queue, waiting_promise, waiting_resolve; + // 餵頁面資料給 resolve() + function feed_page_data() { + if (resolve_queue.length === 0) + return; + + if (!done && !(page_queue.length >= (options.batch_size > 0 ? options.batch_size : 1))) { + return; + } + + let value; + // 由最早的開始給。 + if (options.batch_size > 0) { + value = page_queue.splice(0, options.batch_size); + } else { + value = page_queue.shift(); + } + + // .shift(): 由最早的開始餵。 + resolve_queue.shift()({ value }); + } + function for_each_page(page_data) { if (page_queue.abort) return CeL.wiki.list.exit; @@ -2033,21 +2090,15 @@ for (const type of wiki_API.list.type_list) { if (!resolve_queue) return; - if (resolve_queue.length === 0) { - page_queue.push(page_data); - //console.log(page_queue.length, 'pages in queue'); - } else { - if (page_queue.length > 0) { - page_queue.push(page_data); - // 由最早的開始給。 - page_data = page_queue.shift(); - } - resolve_queue.shift()({ value: page_data }); - } + page_queue.push(page_data); + //console.log(page_queue.length, 'pages in queue'); + feed_page_data(); if (page_queue.length > 100) { + // 清掉前面累積的。 if (waiting_resolve) waiting_resolve(); + // 已經累積太多頁面資料，該緩緩了。 return new Promise(resolve => { waiting_resolve = resolve; }); } } @@ -2085,21 +2136,14 @@ for (const type of wiki_API.list.type_list) { return { done }; return new Promise( /* executor */ function (resolve, reject) { //console.trace(done, page_queue, resolve_queue); - if (page_queue.length === 0) { - resolve_queue.push(resolve); - if (waiting_resolve) { - waiting_resolve(); - waiting_resolve = null; - } - - } else { - if (resolve_queue.length > 0) { - // 依照標準實作不會到這裡來。 - resolve_queue.push(resolve); - // 由最早的開始餵。 - resolve = resolve_queue.shift(); - } - resolve({ value: page_queue.shift() }); + // 依照標準實作， resolve_queue.length === 0。 + resolve_queue.push(resolve); + feed_page_data(); + + if (waiting_resolve) { + // 可以繼續接收頁面資料了。 + waiting_resolve(); + waiting_resolve = null; } }); }, @@ -2109,12 +2153,12 @@ for (const type of wiki_API.list.type_list) { return { done }; }, throw(error) { - CeL.error('Wikiapi TODO: yet tested'); + CeL.error('Wikiapi list TODO: Not yet tested'); page_queue.abort = done = true; return { done }; } }; - } + }; return promise; }; diff --git a/docs/Wikiapi.html b/docs/Wikiapi.html index 08d1554..a176d2f 100644 --- a/docs/Wikiapi.html +++ b/docs/Wikiapi.html @@ -27,7 +27,7 @@
@@ -446,7 +446,7 @@
Examples
// <code>
 const wiki = new Wikiapi('commons');
-const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flated_subcategories: true })).flated_subcategories;
+const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flat_subcategories: true })).flat_subcategories;
 // </code>
@@ -1084,7 +1084,7 @@

deleteSource:
@@ -1551,7 +1551,7 @@

editSource:
@@ -2318,7 +2318,7 @@

g
Source:
@@ -2506,7 +2506,7 @@

listenSource:
@@ -5215,7 +5215,7 @@

site_nameSource:
@@ -6143,7 +6143,7 @@
Returns:

- Documentation generated by JSDoc 3.6.10 on Thu Jan 18 2024 05:11:02 GMT+0800 (台北標準時間) using the docdash theme. + Documentation generated by JSDoc 3.6.10 on Sun Feb 04 2024 17:44:17 GMT+0800 (台北標準時間) using the docdash theme.
diff --git a/docs/global.html b/docs/global.html index 627c780..f10e127 100644 --- a/docs/global.html +++ b/docs/global.html @@ -27,7 +27,7 @@
@@ -1742,6 +1742,103 @@
Examples
+ + + + + + + + + + + + + +

Wikiapi_setup_layout_element_to_insert()

+ + + + + + +
+ + +
Source:
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + +layout_element = CeL.wiki.parse('{{Authority control}}'); +await wiki_session.setup_layout_element_to_insert(layout_element); + +page_data = await wiki.page(page_data); +parsed = page_data.parse(); +parsed.insert_layout_element(layout_element); +parsed.toString(); + + +
+ + + + + + + + + + + + + + + + + + + + @@ -1770,7 +1867,7 @@
Examples

- Documentation generated by JSDoc 3.6.10 on Thu Jan 18 2024 05:11:02 GMT+0800 (台北標準時間) using the docdash theme. + Documentation generated by JSDoc 3.6.10 on Sun Feb 04 2024 17:44:17 GMT+0800 (台北標準時間) using the docdash theme.
diff --git a/docs/index.html b/docs/index.html index 9dd1179..6985c2c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -27,7 +27,7 @@
@@ -466,7 +466,7 @@


- Documentation generated by JSDoc 3.6.10 on Thu Jan 18 2024 05:11:02 GMT+0800 (台北標準時間) using the docdash theme. + Documentation generated by JSDoc 3.6.10 on Sun Feb 04 2024 17:44:17 GMT+0800 (台北標準時間) using the docdash theme.
diff --git a/docs/wikiapi.js.html b/docs/wikiapi.js.html index 099ca86..8266ec8 100644 --- a/docs/wikiapi.js.html +++ b/docs/wikiapi.js.html @@ -27,7 +27,7 @@
@@ -1173,7 +1173,7 @@

wikiapi.js

* @example <caption>Get all sub-categories of [[Category:Echinodermata]] with depth=2.</caption> // <code> const wiki = new Wikiapi('commons'); -const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flated_subcategories: true })).flated_subcategories; +const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flat_subcategories: true })).flat_subcategories; // </code> * * @memberof Wikiapi.prototype @@ -1772,13 +1772,38 @@

wikiapi.js

function Wikiapi_setup_layout_elements(options) { function Wikiapi_setup_layout_elements_executor(resolve, reject) { - // const wiki = this[KEY_wiki_session]; - wiki_API.setup_layout_elements(resolve, this.append_session_to_options(options)); + const wiki = this[KEY_wiki_session]; + wiki.setup_layout_elements(resolve, this.append_session_to_options(options)); } return new Promise(Wikiapi_setup_layout_elements_executor.bind(this)); } +/**<code> + +layout_element = CeL.wiki.parse('{{Authority control}}'); +await wiki_session.setup_layout_element_to_insert(layout_element); + +page_data = await wiki.page(page_data); +parsed = page_data.parse(); +parsed.insert_layout_element(layout_element); +parsed.toString(); + +</code>*/ +function Wikiapi_setup_layout_element_to_insert(layout_element, options) { + function Wikiapi_setup_layout_element_to_insert_executor(resolve, reject) { + const wiki = this[KEY_wiki_session]; + wiki.setup_layout_element_to_insert(layout_element, (location, error) => { + if (error) + reject(error); + else + resolve(location); + }, options); + } + + return new Promise(Wikiapi_setup_layout_element_to_insert_executor.bind(this)); +} + // -------------------------------------------------------- /** @@ -2000,6 +2025,7 @@

wikiapi.js

run_SQL: Wikiapi_run_SQL, setup_layout_elements: Wikiapi_setup_layout_elements, + setup_layout_element_to_insert: Wikiapi_setup_layout_element_to_insert, }); // wrapper for properties @@ -2031,8 +2057,18 @@

wikiapi.js

// <code> let count = 0; for await (const page_data of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC') })) { - if (++count > 5) break; console.trace('page_data:', page_data); + if (++count > 5) break; +} +console.log('Done.'); +// </code> + * + * @example <caption>Process all pages.</caption> +// <code> +let count = 0; +for await (const page_list of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC'), batch_size: 5 })) { + console.trace('page_list:', page_list); + if (++count > 2) break; } console.log('Done.'); // </code> @@ -2066,6 +2102,27 @@

wikiapi.js

//console.trace(arguments); let done, page_queue = [], resolve_queue, waiting_promise, waiting_resolve; + // 餵頁面資料給 resolve() + function feed_page_data() { + if (resolve_queue.length === 0) + return; + + if (!done && !(page_queue.length >= (options.batch_size > 0 ? options.batch_size : 1))) { + return; + } + + let value; + // 由最早的開始給。 + if (options.batch_size > 0) { + value = page_queue.splice(0, options.batch_size); + } else { + value = page_queue.shift(); + } + + // .shift(): 由最早的開始餵。 + resolve_queue.shift()({ value }); + } + function for_each_page(page_data) { if (page_queue.abort) return CeL.wiki.list.exit; @@ -2077,21 +2134,15 @@

wikiapi.js

if (!resolve_queue) return; - if (resolve_queue.length === 0) { - page_queue.push(page_data); - //console.log(page_queue.length, 'pages in queue'); - } else { - if (page_queue.length > 0) { - page_queue.push(page_data); - // 由最早的開始給。 - page_data = page_queue.shift(); - } - resolve_queue.shift()({ value: page_data }); - } + page_queue.push(page_data); + //console.log(page_queue.length, 'pages in queue'); + feed_page_data(); if (page_queue.length > 100) { + // 清掉前面累積的。 if (waiting_resolve) waiting_resolve(); + // 已經累積太多頁面資料，該緩緩了。 return new Promise(resolve => { waiting_resolve = resolve; }); } } @@ -2129,21 +2180,14 @@

wikiapi.js

return { done }; return new Promise( /* executor */ function (resolve, reject) { //console.trace(done, page_queue, resolve_queue); - if (page_queue.length === 0) { - resolve_queue.push(resolve); - if (waiting_resolve) { - waiting_resolve(); - waiting_resolve = null; - } - - } else { - if (resolve_queue.length > 0) { - // 依照標準實作不會到這裡來。 - resolve_queue.push(resolve); - // 由最早的開始餵。 - resolve = resolve_queue.shift(); - } - resolve({ value: page_queue.shift() }); + // 依照標準實作， resolve_queue.length === 0。 + resolve_queue.push(resolve); + feed_page_data(); + + if (waiting_resolve) { + // 可以繼續接收頁面資料了。 + waiting_resolve(); + waiting_resolve = null; } }); }, @@ -2153,12 +2197,12 @@

wikiapi.js

return { done }; }, throw(error) { - CeL.error('Wikiapi TODO: yet tested'); + CeL.error('Wikiapi list TODO: Not yet tested'); page_queue.abort = done = true; return { done }; } }; - } + }; return promise; }; @@ -2181,7 +2225,7 @@

wikiapi.js


- Documentation generated by JSDoc 3.6.10 on Thu Jan 18 2024 05:11:02 GMT+0800 (台北標準時間) using the docdash theme. + Documentation generated by JSDoc 3.6.10 on Sun Feb 04 2024 17:44:17 GMT+0800 (台北標準時間) using the docdash theme.